From a789854c3ffa42cc2485b739947a0bf905b51ea7 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 13:49:50 -0800 Subject: [PATCH 001/551] Create LICENSE --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From eeb472dc18c16b899c920503591f9f04bc574fef Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 13:52:16 -0800 Subject: [PATCH 002/551] license --- src/main.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main.rs b/src/main.rs index cfe935f8d..37d59338e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,18 @@ +// +// Copyright 2022 Lev Kokotov +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// extern crate async_trait; extern crate bb8; extern crate bytes; From aa4f743caff51684b7bcf16ea67cbec2239645ef Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 14:01:57 -0800 Subject: [PATCH 003/551] badge of honor --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index cab6d580b..35a3bf8f0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # PgCat +[![CircleCI](https://circleci.com/gh/levkk/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/levkk/pgcat/tree/main) + Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover support. **Alpha**: don't use in production just yet. From c15b9414d765ad57cf51c7c486c48242ab46faae Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 14:03:46 -0800 Subject: [PATCH 004/551] circle --- .circleci/config.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..1985eb707 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,26 @@ +# Use the latest 2.1 version of CircleCI pipeline process engine. +# See: https://circleci.com/docs/2.0/configuration-reference +version: 2.1 + +# Define a job to be invoked later in a workflow. +# See: https://circleci.com/docs/2.0/configuration-reference/#jobs +jobs: + build: + # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. + # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor + docker: + - image: cimg/rust:1.58.1 + # Add steps to the job + # See: https://circleci.com/docs/2.0/configuration-reference/#steps + steps: + - checkout + - run: + name: "Build" + command: "cargo build --release" + +# Invoke jobs via workflows +# See: https://circleci.com/docs/2.0/configuration-reference/#workflows +workflows: + build: + jobs: + - build \ No newline at end of file From 7f8338530eb710da20353c7737ac7813cd1ad0d8 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 14:09:26 -0800 Subject: [PATCH 005/551] cache ci --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1985eb707..bb7d289bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,9 +14,17 @@ jobs: # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: - checkout + - restore_cache: + key: cargo-lock-{{ checksum "Cargo.lock" }} - run: name: "Build" command: "cargo build --release" + - save_cache: + key: cargo-lock-{{ checksum "Cargo.lock" }} + paths: + - target + - ~/.cargo + # Invoke jobs via workflows # See: https://circleci.com/docs/2.0/configuration-reference/#workflows From 0341f282bda2bbad5952daddb8705cdfffa87fac Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 14:11:49 -0800 Subject: [PATCH 006/551] hm --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 52115f9ae..342489176 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,4 +12,4 @@ md-5 = "*" bb8 = "*" async-trait = "*" rand = "*" -chrono = "0.4" \ No newline at end of file +chrono = "0.4" From a1135354e50a3b46dd94dd9c6ca8d35981da772e Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 14:38:41 -0800 Subject: [PATCH 007/551] buffer up to a limit --- src/server.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index ea6c20012..cfce83727 100644 --- a/src/server.rs +++ b/src/server.rs @@ -247,9 +247,21 @@ impl Server { } }; + self.data_available = false; + break; } + 'D' => { + self.data_available = true; + + // Don't flush yet, the more we buffer, the faster this goes. + // Up to a limit of course. + if self.buffer.len() >= 8196 { + break; + } + } + // CopyInResponse: copy is starting from client to server 'G' => break, @@ -340,7 +352,12 @@ impl Server { msg.put_slice(&query[..]); self.send(msg).await?; - let _ = self.recv().await?; + loop { + let _ = self.recv().await?; + if !self.data_available { + break; + } + } Ok(()) } From 6e89c66799241eca580aa22c73ca86ed013a296f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 14:49:03 -0800 Subject: [PATCH 008/551] meow! --- README.md | 2 ++ pgcat2.png | Bin 0 -> 48254 bytes 2 files changed, 2 insertions(+) create mode 100644 pgcat2.png diff --git a/README.md b/README.md index 35a3bf8f0..58c66e469 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![CircleCI](https://circleci.com/gh/levkk/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/levkk/pgcat/tree/main) +![PgCat](./pgcat2.png) + Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover support. **Alpha**: don't use in production just yet. diff --git a/pgcat2.png b/pgcat2.png new file mode 100644 index 0000000000000000000000000000000000000000..c218b0b3d3600073b941ba6ae4ed6369daf21a5a GIT binary patch literal 48254 zcmeGDWl&sSvj>b01h)_z0>RxK26uOd;0*5WPH+vb!Gc3@cZUEWXmEE49{le7pXXTJ z_sgle@0VL=imBPNcduUEy}En-x>*yctSE(wM1TYWfly_n#Z^HdC_CT{L4X79kf^pQ zfk0#(-s;+}s>U89j?NC|R<;llS1(5h3B=RN90c-Q>ddxE(ra}Ndo{&2fDRmJA&hyq z7L;~=9!S|~U$(A%hESm^Q6ANSD{P)i3f08-dbPgrj2n=tajsfjb*;)DpsFN*mdo*a z&%SoIG4uHHlKgzb@N~aly;Dt zf9Q5RY-_)lgG^xE5ckLsx9-oq#GkevJVUj^suw4uE8h>@Yp*(RFJM2p)2aTK)Af|O z<%~z|nBicrt}H*)o@;LK`}4>y3j;ic3x1u)r=4RHh7rT6nJJ;1clxND&%4ntSSfi^ z{*m*~y3ltv4XRJxkhTI+9<4p+a|j*G4xB1q!3J>kGGM;hMPmyt6WYoO$Ljf2lGr3&zn}6@6P61uFyI- z;Fxfdao3-n*ZulC>B7+0bmN*eW9TIrx%!oNu5;4fB`OgboQ4VcK0in~agwDsj?pDf z_DxPvnxnl6L;D1A8KiqX%^Hu#=8 zZIUcg@hq7#;heQI=Ypz@UE{)ZWpS#WRb%7)@M1cq=ZByS?r3%y_0l zR_qfe<{55HzxK6@rex`B>W-`{YHGKeQ2kHpj_upeo%%>q6v6k&YD#?1F>>YlF5IiW z<5kV8{=-wHeqlGwH4fs8z7mXG(uZL;7PEh>s)U%{!BFe4k{eAI3lCI?3X4h&es4}t zRo}s4Unkc{BoFx>^$RTu@0~pLfNy{NV(XvCQnI_|tv%}pwUw<`PG`mZNAn&3_+>*0 zJLHgs=*1GG#?rM?LH~fI2)l?Ae;?P^6Wn#br$=0Zv>Th%SSQKb%FS}C2?43&N}U=; zdmIWKNN19eOJlp@b`>9bUyhUkL&jR|U9L@=bF9&P@-~8aGWOoMCSO-?ydGih{kfsw zSA%M%hCA@;K4NH*-Mp4OCrXiGZGB)G8)n(QN5=c5_SD-5$j-Mq1Bt!wp@Iaa+VKZu z%x<^VD-apMf-yUW(WZ3v5OPdeP>J}*5pz+_)ne4?AlDyX==6BpQwwTg$JnI!K(6M<2x&Tfvy-w<)}ub*-XVg$|gsja*PlS=2z>wLu}?R zl+J8l5KIrgJUB_WAD@+Q_0hMEc{ibQDQGu2q~MOT{YhhqB6jb9%ctNL?a!?7j_B+W zO0&j$!5Y>&y$+vi%vjELm%$0F5jN!TP31q>?=(MmK>~4J<}2g66Tg#l^&r^v3KQLr zTdyccJhYH|)suu7h^?@w=y9KFENA#!zfk@1*c-haVDHFd$UE+2IgK=J61(s`=!r`q z+R7L1S%+oH)l}4@cwV$|+Z5&$m(&q{_o_RKT$c4w(&uBG%s{B)?vR?I>Wu3>EHB~@=5v6hI$4`4AX?NiT@m(BvTmyO!|rqPAB0MJ8P$m|N11ol zM(?vXNBEHhPTo7SCv;$11=gN;j%;W%Q~vTUYRJ#yZxxoLnM6a$4x*Q)2~5Yty(|oC zCo>8s(9JrIpF9*nf(1?)Qz)DLVY(q24sFbv2=@kY^1PP= z4mKOwlL0Y}M3)9nGA!{Uk!|nANs{JIb#RmM2b{-1R1{(XC6rM?3|njx8jdgStJ3DN zYO#cE?^05xzRaV1fun<389h#54kU8e2>;iYISL znVFh(s6pZk7Dpis1CeN&EAY^9Pd|i^=LbCPEVNu#LaIlRu(;NRoQBxsx>qM0E%&E2 z#5V@*8W1@3Um$BQ{k*yhEcrUY9pq@s5YgpBJaeQ-YDmIenBkFIOL+yiuu!35 z&)ZPmce+F|kBg#Pjo9VNJMKlWmOmV78^4f#T$B8`*26(`qZ7YaqmBpvn+$_{><}v1 z1R>%q_>YmvzG7BXv+!<2U4o$Fdc)fLF(EZSN>XcYVdZ{;uS>Vd>#fS@a_FgbCaTyw zh*5R$=HYD=_&T&^Mfjq~s#X?C&Ul%mrnoHU#86u})|-AG6&n>*fAJs^32wG?6rV!j zxjJJ`w$O{bVNc2i)&Ig_WE8t|_VTPJo2L2524y2@M-W5l|yzM+A1@RzVE4ts-)Kj%DL;Gq&ZoTMb@xJCb(u+Y1; z_&t)NC7iL>g86%#gs<^~Dl;R30|Ifz5iR6Jq6YajKQQ4uK9m-G;uyvI*%Zs?9{d<2 zH*Yko*oDc@Mz@a7$95|^_-nz@UeB3>g&9mH^h-{IjAUA+ajo`tPqadQrlyZ3)|xQu zBvu(=Ab7xviW^104<@k@?h6wW)P}$)3~O%ir?)N2d|!|t8zm|lAOsh+&kWW4V5n%! zZCF31>^W@at8q-Ze#7V_m4=m;7LZcv5-v$|qKakgvbizINB-~7>Dg!!pQsNV&3YzT zMswEaBmyHgu&KW-{?z`!p|^p2uuaPeiuCeQF1rOiDeI|z)8N2B-+dq3q@p4<`4%GA zWfy)YvQ8+=NTMSZxq)$j@DinHi!JO+4(io8`xAL#7lCb^2^N1x?vxK|u2jJ`O4r}T z!&A$r@-?R)*+B?NgbVG4Y7yS~-G-xS?tTEx2WLDQ?6n}-!*<&gl$Jq5Dn1={EUh?; zhal`JU|e$cAb$8q;RuwTf~zX7lfjM#Yval+`v* z==0|Nx$$0)O8ascDAqocHs(b3;j{d;U0oEwhp_*|gzaDF&*3gvGI zNm{!2yD>{N2QGm@X4XgBoYxiyh$?zT*VWnCi=&}7#wtp>FtltaGN9DFC9z+m z@jA>;1jQAKFrx6`#T~Gu%HwS)8A{2C@J7T1qurbR)eO{RoZa7s-6pq)cv$%{)X1~O zi_K!M`_Ljx?1LLQSL2)|de;X;iOt^Ir?yexlvsC82e@?+vJUwruyKq6id0YPhU)Sk0E zNI4JfX5X?=$yk{7?BJ-(VP?_Hyfox0_|ye?m=<1FbKib+N^Xl(Nsh%kMR|V-E6Hub zPNw)RjwhY#lWzNp%n)i(7)wxvFa%o05sLityF`@%k)=c(A~9vZY$w-#4A|SSq#wg4f!BiTG(62K%L1wc>2znMewKnS9BrqVhL4HM6Hyn2RcEae|CuK*yg(HADG3Fcr*BQGyA&h7 z29Eo|6)~QG&O>UA^2YsXfBu1nPiqHJz~X_^+GoR<~VuuJyjRkpAh~bzERck^9aH*TITQ+-FNh1QS9F$^gc&I z=S_MWSEvYxR~+RfBVeFAK&gCyP@WO>T3v?syTP^Nvm%O!eD*OS%?$jdapR4`y3pjU zMB)`H_@G+5OE!o+1d^j+v~74(-!!_uNu;7TI!17?^Pn7Z`3s8jV@4Z+3YKw0vAhv& zzl!3h6pHf)c%4f{il@}E6#E>RQy6Rt|L@~q&2NvY%m|v-td$5uM&C;pE0~a`Cr2l! zp@X4k@CSljPd4$Yn2-nFmj2LaEDLz_Kcg5dRS1@c$vZ(`iUdmYhg+& zdFJ-fgnW^Y^txGKpnS_NrMV^j!9~E9BSmi(2eUNaJtaRT!WR)&tC3FDAX1cZ4UM}d zc|abH-zQt!o@v`E5?&D1E5cCHEtxVg#5VBHw0Qb28oEAKoXfyA_Vl{48Q~XKPDYis zYR}j&WyVq?o^l4WF08k|KO;h^?q-Ck2x*NeFMgqm0qEp*!bW=b@7*>JF1EEGUQb+*H46 zL&m6<`H{0$HzwoYwqZ#TYAy!l<2m@%wgxixnq(O6d0(zeH~ zHv0618~2jO2O<-=(ozzADTfBO5F?txf@zahV|avE{iimt8;>InjGb_j=&7EM$qqZa z6LyaD7%O{Jhy*BG4Wq1lIoi2Lk<@0^MffX2IW{L`W$!}=)hB(_n@msup};sNi1+8+ z1}-fr76q(g20jKV6F-XP#FgSwV5CAsKbZ59TMJ32WkdC&86SNOe<9VUfp_jwIaAPJ zC9&O`v6!}3ye)p%b^o&Q0XL@WiKR9*VK|B>36Z1+Q(`(yV>DkPst{1wAKVoUe3rum zl-{2fnoe>fAgj{W)Y@ZO5x7D1(v?NYG!i*BX1tHH@9Bu;ptD!yVL$d_57JKKCn{@j z@svQv2&a(=4oO2So5bj1E)Xt*EKfX&k4nPQNa)xHW{b6J&5hKHI~sQjajrA1H^tA4 zMD&GJQ(8AP1BI^llFtXH3!|f(&<*y!n_Nht;g4>b(JatI!~_ejoZ7tH#fUM}D-sBm zEE@zZHAw1g_TS+L`*U;zNG?c)ST$ylvPjLYht5S}vCm?QEV#~bGLaTgnjNY+fQUt5 zJS|puHyrpY+0Rir zvo+WsWN}bV@2r_K96VKE(o(NVr-Td07fXKJ69lk-Pcm4=I>g=KVk3cCFrs#o4`>nQ{pGQcwNooRkqO2mSaw9Lq$37cHgD3f$as(Hb zw(U=>+xD$MdS%Pw`d@`#Iox`|s|O=I(2A%c29nTwZDVOWf<}mEz7?Mx^Jafg;Z?*j zn|?s%M5YQ+FbYW)Dy6)w=-5U1L1T{C^bOsLB268ls%M~27QJK})Sk5PLQ>2bs;xW) zvp7N~izV)`upOpNZySszXw_cEFaRZ(@^SRzx4bJ=$|&!5@YpA^2R_DJW1-S$2{2Bc%OY6B@ZgVBge6PM zCz0C(BuBr~9Z0?vB(1&Ck=YPpvyTpC=n_d7>wUMT@coSg82tgd)0DvB0NDbrMg-*g~^M1;bmj#bjm z?*hJH513W1M1G;+mSJ>V>Oo0J;K`0D!ACyg=@7d_KkJWjK2++jE`JSmAWQ3Il0ZL_ zXfC{9Lil7qDNl!r@FAI=#7%k1UKpjt-J>MG%>h4RyPT14gN%!FWdJF0r#W9x`rvXr zLW!htyz0}ej;$QyU^JWi_Yc(cw9-BJu>^d5Q+^5Cf6{QbDh&3zQ{SE48It_yr>BkA z9zZGh_z5l4hz!k?CJ{IN7WKWh7#I`QdWrR{cJ)S+Mlp_+IQ&L)c7Mg^b z7U|+Gu`X5dxW86}(u@R-k76FB)Q^rt4DH24sgEv1>Bn1R(40l6Q_MGn%sgTAmMyip?Io`npIk~&8v~nlb zVhat_8DervUGQ~*j6~hj#=$VBAxk2b3i2*Th#bwR6h3y(Ie2sB%p>ZwYWy1NuP6$j zfT@L8mK>iDH|Yu9hT^nY>3)3n4^c5oL`S2x=hpcfjnoo?-{_<#v34Aftd`!GAfBkY z89^(G`e=N^+EY2=@t?;C7JlZjZCh9)y;vICSq5Y3E=@t75M&fnS*XFAyeshwq%;Z6 zHmX)9Cq(E>*>>+|V@W(TJR>x+bqy6o8M(!flYA@mMe3aq?Z`y<+BpO5uI^kAX}YwK z*pNabg1oGs!hnIOboe{Db~T%0{$A2=13yREtmy^$e}=_cgiE3qrdb(r;7b+#xUN&& zLmAWbuQA0lDVqoHU7R5kb~(e%m2f?wJDQ!g%<24qqGJ^(#Y;dlf`5vP;~}?J+CMs= z_o^}^W>CN?hFXM~BlktgU}3#SD=MM#rv1eX;pf_;3f8N*-Jdn-pH%urd?;8J9T#yp z_9ni_-mqE3wLRwdzRpo5H=L<*C_+~OrI_iwoW6-&HP)8OdKS3Pz`9-p^(bR^j9qQ! zS6mz{QcIR(M8P>i5ACby55=mbk8kzd+N6XLIzPWNAxCj|3deAK%~uz`KH@;7d>bWB z#A_-ti=4Sc_a$N#QK5wC!+1eaO;fV0SU(vlY$p{X5ew~scygAeV@T_KB!_*)K_JB< z3z6uzw_mTg&bW%g4$=FdWvsy@Q|+_!(}{HopR@=;?r-_G!oaq^yE-oJ)}94$&Im_! zhD0^wS9ek@hm~=nRiBB_i9N@EOTq2o83wOz;4~(hS);&4*-Lb=+y$ZqWTMsv;SXjA z_t;hNEy={Ld{$H|b!*fK#hesI|C~NB{7VZ3V=B@Rv@9|VF>3BRC~YxDQCX`$n~OCc|uvK(P|1{2Uk zwWjI*eDUf>Ol8obv%u!w(#>Q=vAl$VqK@4A)_toR-bB+t6n0L*2J&E0Uvy2ttZ_rdb6@k z%Lii^pi`ZLn%ZkcPprk1)A%LyiL3Grl}l3jhsg}?5&1wZ5kBoKEX(~3+MNtv&2ybt zVQV|0&3N~;tR_J;dT%~36w;DTpG=1|7omE3ufdUj#w4`<4*3hG{SUHM%FkskgAArXgtV~5F8p2lt99RhHVbVEI> zcogw#Bn_k&J@TH6yevJ_EyB(`s;3Md+s>`PhbEWe_lR^y1@6f{!8D)e4ut|c!K)WP z=Gvy{EwNbQP8u!xohK$ZAZ*esf?e<(1-E7XI&Tj5dSWm>a zjge!~4H+O}2n`|>6KWp;F%y5?lJYF|i=f)N#PP!Mm;jzAtialaOjZ}Mff6vsa!Ae4i#P}B!VV6l-M8huVEKu%DsrwVCN&s z%#@JQn4kAY!(G8*a}PM^_(%)R)T6AyV>YD~Sysd}&(|gDhfsW@bDO9eU|4VG@NvY? zDEe^KHJ-Jh!5rF>VMyz#e7w-`HaJuWZSmkk5XwQeBw{{0%%-)|RVA68Kr1GH##?;j`e!#i;$r$>X^2oe1JOvV3oI$AfFlfS z^R$dsyX5=9sKR}#E`4}h^cowMfYB8+WwCb z9S-vx>8}l|TQ7e?@?VCMp^!0lMlLd+OU94RwF~BIzai2`W1(kAtZYTcG4ti1K=(Sn z7QtM#q@&u&%*RQO_st9Lc2ibh3>Hn|?Y*%Opk=KIQQSC`m{&eJfF>cvD_FYt$?z%E zxvb;Kt@XMV^JcseqGU3$7=hL%HY~lj^WJN#ffU1aNvs`83!1?!^2mvHvdFZ)h5AQ- z@~3h7so5XFCdZO7=N_VVaa+GDIzLg?mT1N)Djvvd4u>L*N~ZoK)u)sQ{0+rj@IHPr zRblH?!*#L$%LhHE=_BoP<$9e0?1Pbd*Z0{int(U7?HmHWA zTvXi1AAJJWul!9DNgw*Rq1;lEnXk2IR)<*TP=~)ApTTAgO<*a8SoQh9#+vUB+}6p; z{vEpbMoWko{$T~5V@Wb-DAxROnJ5bCD+Z=`+-z^SaJ*E`I%rsDq$EsGRO`$lKfa4H zvYLyI`^=mwMnoO@1^bR*|0r04-)okNb4*q`@}n$|Sk8`^tMYv*7dpFjc*?v&nETgM4qDYVAl)n8# zB4`ke5k-=WB28Rpxx^%_-Sg#u$gjoju4`);`#_uajj|!m=VZPP%?QP>a0~KFks?o& zr)?x_U!&rk%dCw-_m0tmx{h3zNg4782ASoc3ov&9$+7&6ND*AXC#*O{Ec%5eXTq}n zBDZgIT9s(l9tx*{W}R}3$aZ>QI&#)3+I`GYcnkv2cYPvrtg~YnEVcUR7|!%9Kl#c8 z*d-|8$WLD}k0%ZzY9`-VMBFb~FCN4w^ zuDWq^-&S3d!!9^Dc){AXp-G?s@-pKGqp?05{TwzjQKFw`vM~O>D(7ekf(rYTj!06= zj#s7R@EErRQy)5rc&k5U7Cq>}r@@dmDy)oNgDw1TM#&dkon4N$TJk{rc+C<1=djSXZ6KEkRWJ`TnqIW(|s$f;+Jx zD9cdOa}TDp;4JX~jKi7t3h%^C@Pm+dtbsaJVsBjP#?a6Pi?5(C0A6fcB87&=n*P9r zUz)7t?jwqUO1;u5YfqkbNcKTh&C$UF@E@4o{bk0tYzfxorOwT0U$>o$TurIped3ay z9dy=Hj!&0rL8ATgbklEoEdO9r^5w+so3ZJeM2E7YLcYBVXo@I`2$;n7KplGNK~62u zWi4zl(U@YERp^@{-Hfm(@n$Ov=$dM^U<%J96u403(wSOMTx$8$e)ARwzakMnoDQ`y zZFtr>{O0m05}6`Z7`X}eqC7#N8&85oN1@!O)nP*>tS0wrj`4ykQ^(B|*{wU=yPR2T zCOwcd(;w78lQbvIz=ja@_dGV>VCdFM(*|U0&60_#`afSy_vEA=(o+V=>2!!~EuA}D z4ng~|9_C+f|1Q;_ z`^<5j_x3fA7F<@3o{dRWD7o1!-7x1nC=Zk4n8qwVlPS25gBs}lOg@ol<{isx>9i3^ z$?DprOL&5>B=?SA_`O&f;uJR8UJR#AVfpZHL62E6lO~?tP1J&1p>6Sf?f`4h($s*FfnK6$Xh#kWsLzDVgKc|0XoN`+af4G*;g+_00Q_5tdHaI0ED@DpE2px;J!+7W`u)3(rFM>^{^N1S(aMcZQ z-SaSef-(-S4I}SZzM+2n=)vLjwq*tCQ=aQ+0^z8V8Vyd+9s9#qeil#U>#Eky#yh76 zkXc#9?)!}mbUbF2+qU6$pLUf} z;0z(*VB%n6VU+N+a%UqGL?YpLHZupSic9{R1@KLP%+l4>5zNf&;o-sL!OrC1Y{AUR z%gf8m!p6+T#t2X_x_H^U8hbL@yO957@h=|Y5EoNtD@Ru=2YZsgJdI5p+*}37$bjc0 z{~@29qk_VJvbT5nw-o?;Fnbz1GP5$VFx%NN|Mw9tt`hD5lYa~J|2V=$9XOuFtO{{) zaC0_=NVr4nUCIAD3p3OI9PjAnZ2ONoW~R&#TZkP%>H^Hl`ah-WB{l|T)|A*ybGc#r3Ha6pC z0sN=6@^8{5PimLtB34|1W#+|3mQKk^nI7 zU-y9I0^};@|4df@w%1=1{=fM8_d5K)I0L}?|0elg+4uj<^?&C2Us>RPMf`u>^?&C2 zUs>RPMf`u>_5U+-A^lgm1F;7XkOxrA^e1J?0fi8piJX-9KP3_9UaUO;xI%Q4)^P!W zkTL(hp+K40c)(3~R~ZEf_&sP?92R&ygVFPolw_yYBk-=0k&< zwF=7RI>ymD2=-mD9zu(e8j28xBLZczFb0M$G;Ko~fh+>zpzHf>=pk5KnbeApW^zrL zb3UCvPd&yv0<_8Cs%>SYos9*jT!`^Cm)iM!_V_HOvU%UI;Gw{U^cwyD>4y`ns5a(D z17+D!-l?iP>eFW#g#5}Bf9Iy2oU6k|#Mb>nmMo)!i54EB*;VuUSoRG0&tNckiwG@T zGl?P>9XdspvP_*WyuIBgN3n#jO0`V)#@ml**og>q5{jPDxPfi_h7M$}Si(MFULU4g zHIJyTukWN;!TF`4g1)1@oo~^+-inhTc4vQI3l|U1la7YQGOfM+(LHWEQyZCW+B^n5 ziJ~|yE$!9me64c^jabljWMTrFj=lncv@-`kA_AeLtgOsgS~`M|kdQ%AN^0Tp@sS%2 z9)ADv;Q`0n%WL;dDQRfBcC>+}=qVyeTW6<0Rbe4{@z<|Zt;NNZ9W5=3&DGU$>+K#j z%KG{WG_Y6_P{U5^FN6Z%pla>fKL49JEn<8=_mezWk;TCdd#a@vW69Lr159egN}ruX zsiY?R)gk2IA9>wD3~~`sQ3Z&Ih#b7UzgWGm532wC`LmUto*vTG)zz*+H>UzR4Mh)Y zyriW3$sY&`?`LhTst;l9nc^LuocLaEh7cg> zs7O7YfRm07uxK<1LO23x4sJj4aqPINEUau9bumYv>Jig;-K2ffr)FS_+>vgSX||MX-!R@p{6DmqmYo0?d!|4J)qKWMn=Xj$+vPvk6T0U9nkZ< z_A*{^(b0cX@7b|Wo1++7Xc7tfdM;e40E%Zczb^w@eZ=A(~Xwgn6``gM*U2t!)fP*2IU2i3xvz3AlZA^4F<* zbwx$i`GMtrW>%J0`|6rc-1+kf_Q`9;&98vGXIcd4r{`yEK+s<^GipvQGvfrYx`;?f z&YwPga`*J~bcKV1;{rU2c<%THTephK5IqzwZu@&Ex)om75=oSatcAt?Pqh-oiHY&? z=*4=Acs&)Bck9#ByhZE#`}@&KLeJMDvt?=)=%Ngj9{ODfm55AMD>gPZetdj<;l7S` zc6R-@r+j~i__{8T5Ps+J_&r=B3ccP>1*BwWcP!X)1yn8^yMP){!?Yx6=RgRDM@O03 zIy$`dWo5LD4GkP!m6h@y8$D18Ha7HH`ubwoiqu#Vy?34a`I{VX17EweCY`dGr>+oC zQS)D3prWZ2WZkH!sC2itw<|Lh;tRxFZBl<}fRggk4ZE<_@YgFy4d@C;} zXQT1=bXKYF=2tkK2HnWQr^5TtG{w+?MPI*uJw`@G#ywr@zddt>#H%#7e{cHQzrMUV10pvg8kbpCzr*w5baZsohPId( zy0j%hTSeBbj-iB$o10m{>vBhGBmS7SNi&V{jT#8+DiS@bLbK>P~ zC5YQDpjOJo;&s{kE&(`4m5id|8m-Jv-Q4{vW-RM3$>nA`xgQM;U&o&x@21uptkix? z=J183rfND8h9{z*7Kor}O7ZJF2o_xxF?wYn?v0{!bUfLG2#rn|55w zgNFy98?oF5HLW-Edi6YU6{elhKnC&@UpCzyihZ>2x=$4k?{`wE8JwEJMaE%pWYTYI zF#cI2WBBm!&_$0Rnpi;dVqSlr3AmaDpT`+tJ0R6_t?6)UH0ManB7HfDQC}$HSmT0$ zX$HH6;$f*up`fFOhvAgd$7l+f$f~NU?zLuz^@4bz7apVGIETP zO6^2RghR7`i=>y5lateKGB`#nS$L7uLKQ0+5{8T`O|MmLo5f<>*VWe6Hoe^J5P$z? zJD&6A26bobqWtkW1|174^ti^*%CN=JSOW-q5o6=c>$ie`egr(;p7srXu=TmPxacj( z^LO@qxISb@B-Na6PX|Ik3Vj3+`Nie5abZnYSJz!efLHHjGK%azRUiDzws)=5D=pjTjj*?O+s^j3+9XUCung(b$2a#G85t*TsTYNK z-1gZMGczv4Hi&;Ws!#j(KLI;8jj_?s4&wnk1PLu|nE`qzdnM;z>G4I-b;-%eD-dGYMz?X1?nkh(CFF+w$s$O0f=a1)}H9Oou zi&LcQ$zmV{^s#a(Mev?QOJ+#v=_}^P$LX`|e0zXNe!h*`mZg?YPsq#7oy3u1c^JGtoY`^O z9)i3nHTh?f%e5v(Ha6#H4K00$X;tsRFiE^ly?I7oe?K?6=pqUirrpqBj}gmBCYPPA zhLI784a$&LWMt%5cJuEv7%-&qjc|V{3b|-$t>nzj;jXgr_94B&uyNCcQUOC_-vFTU zQ5=DEc53Rpo>DHgA$}*0I%XU>M=^Xb8dvN1<^lB|lQVTW;sQs0s z@a*jD9)E4-COg!uEbBDExex&Wq4!?7%;t=J0H;IG<@tz+h-QbD%};1eqDdD*%zDu& z^jcgGkB_fu*x@Dsqw9{-mBX2m^OmD>uSyp-VcJ1d?iCUshgR?MoMG9#16)9EYPm^5 zT)e~UYJ$7DrS(ud$|kFy1IPr?dPGatQ;91KV_A> zU%?YcPa3L?K>E{2){-u2%Wdy_M*Ml(adkyqz00ET(@d?{isyFh*80osf)b$QIZ5au zoYgoW*B2Vu7>03HU?Zvpa#|$t`9ZT{uDzmSyt%V;wcon&vU5uwA;_R6?dj<$Mo>_& z%Dc0RoPwfg1@Nj;5(H}AIC@ijhW?PAOrwp6f&)Em?UviCDIq0(x1(f03igg^OA55` zH&mbTldk){NOBb6xw)J18g(UBR|U{9F&P0wJocGdiAzdS(j71v;oa30pGl+con_+^ zn`lIBx8#&l_Vl6a%k!GgxdOk_mbj-y$wfhPVqP9|M@NShfUdT7cZ)4K2ozHx)@&I7 zUY-D8mJ=gmRST9Hj-Q|3#yPRDRBY5|0RaJ01ZXHw#hmQ%(NRlgCc)32KV`Egqxn); z%)eWD4-<9MA)}$aEDOVAsnXFzZ2`zrvazX2(E&hlWu>Lk)m2qD{NC54J~lQL%4%xk z&CSgsSd?3iJVuSW0Y(eKB*HZd;WmxB2d-cOVjXrtDx?=xK- z%2o7{>1?TV}*T@cO3@nfd90_|;Q)*opkuQB5{~q1|{W zCgW!|Z+-F3j+vCFXG<0!YU#{jsciAjvAPpi9>S<;OZB8K#?-k|cna*1Sk?G`8ah4t@ zi}`F>j0qHF>mfA7SOqk}+dT3)nZ*M`L!;Zn@y8$7&E-u^OiT9tHRpVC1PuY%e3$R+Pm0S@gO#eV zaA4g2{YyuXYQ@|U4HHwU(ESeP=Bmf~jH{bl?4nzZcCD!QUPj3)CI*JH5b1z_E*F<{ z5=U)WnU$B5)5QsZ+a&gOc3J?87_No|`%Gl(Edi#WfN^rF1^h>C0NAKp#KpydtS27X z+QdTvdf^)m)0 ztIFpIDtP_wg|7m3oF6T%t!2QkGJgOLc@gqfFd^6HjJG)v>HOfvoBz*-!)f!1b2JtX z&R6OXTbA&C(zE5}soU@DQwJWWO9Cfv;z``?AuS9Y!FDpA3V8o(ZWqdyu9ui z;n4`Cm)qPb7AAArL(k6aQ5r;dl4-5Riex?y?Ck8swY0Px_4M?Jy}Z1H1BuUg>Qk?I z(=;^ZDZR)5F9k>*p*t-_LqUk&%RPp@DF;|G9@!6z$#sG zE_|F}^|?Li05&0Rpmr8RaPmF(3E{S1X*s@K)`|yW?nBe#w%3Q|b^olZyYAOcLIQ$v zO0h5MiX}5$F8hiP2Zixg0G1D*;{tL@p_0H&vfpN3ST>L|QF^a5kB*KAr-WYl?`i@% zi}5ENm4t;sYyOYx@cUDF-M=4i>S&9Bu7iscpma^M{c2{LaX;K$2>e^8H(u3iYX<2o zq%ygD9&r#LQW}kojl^_)CFx<6bavNN>($I;)ufdFP$3jkWbWPtN41fPw${-GQ^SbJNh^ zAnh~nx{=`8+S>Jbz-xQIC5P?-I{A2AQd)BjpMZe=hw9@z9;bpjpF8_#hVe0F!vH-! zy*shD0%jgAGc%iQF9`|C4|7_EyK1t2Q;p0jSkxL?S~WfZ=HydTQ^U@gtTNo+_6iyr z8lnSmE|#I!Zptl?uls>AcxvhNwl#?Y_I98jKPIms5ZGBu3TWzqq-?O*XruK~KP9-_ zXw(axfgV?`e&p}g02xn}G4#KOxasLFs|oPdhpHZR+ATMc=jR*uQxOC_%;ecpeIAt! zN0d-W&f)XWEhsDu_uYt$jn!~JSu`d>>pd%OY@Bl)Ci39Zlb0=nA8$8jnPlD& z!HM4eTR3=d1E~5YYyD~H!~6FYzdIazCxHT6iP&=k%E!Q9&8hyE@7rS6%l7jwz16%p zzVxXH?)4K}D=XF+V@WKDi}lx+=3nFEHY3%mGc(a$_j_69Dp)!Fe$TtRyQ$aLF~(kc zK*QyFlsfOENw;c&FK5z5DVrzp8Sp{fkl^4)&c$J19q-lyy8Qs;u7Qn>{R+@&smZ?4 zb}(bL8|s1mC_x*`MG&i%$LnIQNK%QD(c+GTmf7N7qS%=J))y)n$Jq;N>b$3CKO6-B*)*Ye2QUH<^YA^lGRSvr7R( z53rdI#S{QhJ;l$@-?hV29i>19fK`;BJJgs@|oLJtSz=Rka1aOthv z&>(E0zkja)_@Ru}!Vg8!vAUR&T<@O`Kg8aa0gbE(#<8)9iAF|4|C~w7CntQ$4<9}( ze%nO0(bP;`me%*t(n`CT81OF=R=&Tpu#O@QRCzP#w!oOS=6j{k4)l0qHg@R&|1Z8-vPr{} z^d>hKMF5Z$BePypr616K_?4fZ?|>YfFz%FHmB&?mSnCwKYJcanbQc~OBM1RUf)zkFb|zX{THk)zq5+5}o{b;}pj7l<(C|1{rRq8BoS>i}h1}d+3luo9 zdH|)!bQ%~2)n|RXZFS92E&rPQ72v#{nwG{LFp>H8smX46tYR+nomDu1{Ud6+U%)^! z+d6bPuiCJC&H4F;8xOAiC_y#q^=aR39Khu(BM3EXR#sNE`T@^3M;#r0$D1(8B=dU2*x1;j#>N}IZDOEf zt79xl3tQYkjr&-c=t39|^mP$|ynuj=tRWdoup1K*@eRnt8dBoo$$z_EH$c}*8iAHN z6mRMxI~(LoQydOpb@rK=nSgq?PqSiuC@j11biFC7W6t(k5(1xy z(B33bSn2xtJty_`^_6*fdF2CLtIXhF%bP~nO>laobO4pvqbSvk4m zY!+itD**b&0fi2i2zuOghxJdFmr0=OukQkA<|FjqiqwzE=MuC zSS~H9`)QYXJn8f2bq+ifVhmAm7x(XSYzcfwqprp5?Cf`1CMHE%I=V|C&);A4tgL8( zL`B{Cc~#6Sr}IN=uaU6zkvIg9YJT)H^#UaY#R)Ph>a%LG>Mn-pl8$dq+jWg$4NIg_?6&C8ayU@9W&0oWFn;V_0%B z7KEkZEdbvyd)~a2cIbZ26M^`B&*D}xXuo9#5Dn_$8bs$SwPH4UXJ@CCy87%F6P5e> zd;JCd=DtE;cO7|LPG>DVe7;*>uXWt)n|gkJHr(K;ZqNeJ76&toi*RMxDT;~p{|pWe zo&~CbZlH5mK~GN~IcU;Jb9;Jt7`d}U@&BReDuA-;x-}q;gmkyGgrszrba!`mmvkfD z4bt7+0@5YjQc8Do_jhOh8HX9dH_q96t*6$0Jwd7Z4fH6rXd<3lXC|f^z=^4tadFU( z!D`*~2MoU0ebcvPV0AU@c95m}<^JJe4(J#30D}LA|B}F^q)()h1)8E#Qk^kI^2mtf zFUM!p*H|>WkxQq-!ouXI(~hav|62I`@88`Fkw#R-D(U)-oU9^xTIympV5N z^!FbE*@+uAvBsuO!^VBXGj#xTUAyIaQw1a^N~we#-&;**|I6RHzA2xysa9;7g+Bcl zotQYey}Bw~)b|f+C@dUcNpn_FRsDjFff44C|DCk-5J)A(;9Zlrb7wE!1@n5H2!W$< z{#)X#9d0MwFF?@K#=yY9iV^=B_K43FTN3PmNh)yo?&Se+9lE!dGyjRC*%_I54Veyo zdwk$VC{lsJqRFf;*?_zh2Y)~FVZ)CRDV#}z7GlN_&f;_Al2gQNVi?nsN_g+qq01zN zzDbdT4;Y6X8@?~_64uAfSOHc2gG$6b=WbW0?{aUlkecTMlCtJ?*{TH73s zOk{oEn`CD1TbmB?!Woq7CuGd7^=C$W zJd1&SF3;F^d^;s2bl>~!usHb)wj-3--Pr5MrUDveuwMKfw|c*&Yq!`TFSR-*0yYhI z{p>}J-=u}?!p%829H+FiyPMpAls&J42JJv7vA(t@+NAZVT!l^%4LV|%D}9&imtz)` z;L!^wS@0!qjRPU|6)zJ~An64!lOrKD5L0+S^@@aaVg6!Puuj7Aou=3y7f{R^F=ZS9 zS_qhc=PqMtbLCerfue&#Y_S<#Ofh?@gf%h6_q7@f7&27K9_HreMy{@|7P`9GPhf4j z0_v;-EEVd=mkQ%b8btm#pX^%0fzUiW)*t#UJ`a~|4dzpm*kTu-gwUMe$@YNKZ2bEC zFbbIV9YOh05d%G%n4<#qztdB6aH~b;t&n30zx0Ek=NKS3?_#4O3%CwF2Xl5_-n|-^ z%vsVhFsR(+><&g?xBx2ndDt-s7cGoKj@S6-dkLMj(iW~*9%0%A@SmE>%dKus*E`Is z0K}1ZbK^L3;eO)Vy5x;JJfz`vJJtXm#RB$+4~CK9;iVVX*9<^cu?2WQxNv6vig$Cu zF{?o$shC(o9GVnE*hDydmQ`?8Z~Jr1v~A*8E-jvb za%b-41Fp=~a#>J&oVc@%Pu#qHzJC2G4p@KcVR6{U<5)fO5r@OPevGH+@|4?Uo zgW1H&*wyoQGcX$Dn*Za68*^fS6IgOGBbxCQ)zxIcE_v7dMKdLNe}7-7WPI7ijVoKO zL#={Hp+ho3bkcXg!pdsrqf|j4bFDUWx{O-bz&8K9*}}@oql28>X1A2v4I85hl*2}`4gX!qg| zNt#6^CnsO~%*wi}R;j^K*WBy|jN-jPu)EGd5hQRsS((U``9b30RX{@yti-9w$%{;O z>-_~jmbLs&IX1R=WvK*FA+*P?)=;H#7Z(>li9duW!vRc{Kx%XYeN(8$gn^0a02oE3 zVZ*(jUqu!FOk(vH6p;5B)wx(UY1Kgb^-NDsONL&09xpcxQ&3a8+qt>9zj~e!y$|Pa0NE zVd}AyGc_GrJ3c;EwEkT=1MXTI=C}2ZyY~;jcw%?I=jKjnGhh$RIU8vDAEew<1`DR_ zK}M;os$%)yJMb%3z^NaUfz_>|q7n`anRgk{Nt6+i3D+lQ>gtQ%zv=Z*N=^3n3(xXE z@~K?^aJzM#JU@5vx>*pwIGD`VP6c2S0}ZWqU|?VuBB1@sR&&^(;3e^gL~ZxxD8ege zx~CrUS()(;CUia<8(Y!(=4Q=ja`K(_l#~=&(9lUHBsaz`c42%}Rg?LE4G{24s%Q7% zf>(gF2I!%lK%QGOfay?Qa&mTN2gctAU{v?W7ttWbo9<-^xo&N)FE7*X{rO`d?zF2f zgyyjBIz|LUbo#i+$Q4l3>$1^iX{6A}f;~p+ZBEiR>COy02vZPWRsv>7_C1( zbKv^9dU9J^US3{XU0p4iIQoY=(&#L9cQevW9=?#XOX27X9HgKZ_DkuBTZgx~d5Jz) z)7^!IBCh9KLMIUz)NS$IJu53(38|@Jz#Nf!S-b!J(W=QykQ|7tyU*8i^8POA%S_$( z!p?xfF9BNy_F7{K%c^UGdAc^y>2R9tae|ZQ;oHwX|Mv-$+k2_{Gk`|$J6`@7MPOlJ zY58~K^@WRTY3Jx@xop8omM;015PUq0(07pU&_oNi6GE%xom%zC!TA#FT)to>VQX6@ zTdcGUBjj+-d$#4-Erk{=E`-MInbYD#Xg;*e#Z^JuFZxt}QPgF_q3r}Z{Tk3XDiabC zW&mPiA>{K?G5(z13buxlva$%Qa&isl?)DlK9}ZHobV!r+V%sPE?zInSXw&u`U)29R zKl>Om<38%z3oBx_+pTje>FQn;?)D`otC|2t-wMbx?XcquH_6mhmpv>r1OTQc*>a0# zoLNAUe0nK`k(qR#=#vujvjGrdoIwqg1(rVE4St!HmmD;{{OK?Qz4@Wv*|t+laf5=Rvl+s-vUhq&^^$C^6!NZ2={+pzdHnbV?WZ z1FQ`{?1*#BS|Xp@%zHg8(mp#|pDQE8k#k_!v{LK)gyOeYz0O}&hJmx;d$(Q&*5{L; zV$uOuthFKCXHnaUjIBT;Sz_AZtn+HU6Cym8ebiq?xC+A=R{L$@y zPwxw4Ny(NCuZn`XBb5Urfh=yakp(pW zp>YZ(ra0cq5sFBFah$wmkshgYTow$Fkn-`_wjOE4OPdpv|xjgyk5kruIN3 ztG?wVWk5gV5-6)C%vgaaxcTFUnS00jPXNNofWf1Ba$>6k{2{q)ZduV8gxKASmxx8{ z>W8-LCF3}vuQD>7nY*n%DdHod-b=hT9qX@A;AK3P4U_njw-yxzOC~@|V=-toY7p~r z3&lo7HQa;DoTs<9`y;4GKxN|Y(z6Gw+#DDHi6G>)YjXmN0YL!qsn|scRN--;{LcXI z)DSyr=)O@Sn}Dse z^OdRP83t9fx>}9idVEe!HxmbkTZLZDCi2$%$sZ-<5Ur5s5P3LaGjk@N#a!IoC00+| z>>HK|7w6|O4LLITz-E32Oba&P|2?Pb`wc0V3!rBP56BECHP$a{Kt{IP6W(E{G8RhI6M#o2mR zZ-|8_7h!B}YI5?6)7|O%sQ2BeD-$E*K0O^BVCbRJ#od(+vhZXhzkaEtFzMUn^0+fb zL`VObe_86BHEYn6%i)cX=YJ6G0z_8z>({X%k4X?{Ea6;p2r>qw*~!I)twyb)3FnbO z5ffmh(5x&%A7B-^0Pgt5X$p6T3q?u_!vct}z5v_DfQZ!c&cVSQ)v#m9)31RaNlsiH z2(?;@nD$Xa=-Uolfgmo?Ho=SK}pk6k2z@pSXv1BQO+%JjXP3PQWC~K zg|6#+aH$bZog-)F%R^$!w684T%GQmUv^W)&QLy?=idz`(6MR;6n ztl#G5razG~9@R=}RtaDf_RAn+l&k0Wuon)}c-u_P%rR_%q##mx=%O{WxG3+)oedl= zo_I<;koih=@&-K!KJYx7GcRYtn(4us1OyoZ9q-of($p$j?W{dK?u~(#`5ewKz>z)uv_BzZT02d1NlGi_As0jSI zB5;y&ZC4tPs^Ca`m`m&GBAPCkgFiQFAp?a582axxIr-qfbZCm`>JnXE#pQK$5Qx+z zBsgM=Db@m0=OJF=RV67U#SLH@Z;cA(nQo;{o4Eh-X!>1VHUkjy4SgIJ3=8|x!yeR`;W&mO@PxU|A+xKfga%*b6`}v9!GW=TK?8d$H$Q z%lhrt1AWl-)uEuEh^c~WrNqEOsbcvDnH*lv*@77hOtAPj1_ne57py{@zg9{oKvygv7S4nNYzd0H z1_Snz4F?-1Cj}oLQDni^`S!p)XqnS%&jVQwioDv2E`R6C!lR>k4>sU(XvC=nMQ3jK(p?AIO|)48MArA4c3x3n&T@M?*{~-RatN;!P0|L0EDLiW6Kc|`3WrWds|)~u3vuvZ2;mXc4R%a`mErjBOwEJ)CvfYYu=dy z(};GN7s4Jg{1_o#C!P}@7Z;*qJ@!AHK0*hSA~zV6H2Zk&Lf_w|efJ+R4SvZ~Y_HI1FaYc+@*xUwxc zwWjCogj<#oqlRF^)*~1zKz5Yu=OPjr8JW6@PNotJ7&8Dq&+;joOd}JL3PcXn`D%{; zJFbpaAfI9=Dk=h0(`jVqw$|#H#%_%bv;c5@AQUk#o;Kfzi;Ii7+>UYifsIGV<&Zbk zk&-hVPi_CL?t))yQZhx$fkd6i~yu`sf6Lb27qg<34iMhaoTss5l=%;{|+KJEKHxJ zV8W5muJZyGeDhWVsX_-04GpG8AV4PZeu&BM8{lj9tgIl}4%EgvP?nxSV-?lX!k@Rg z3Fy&2T9)iLsViK!_%>t}Hg^9IIAyDZWDUk5D0%jFS>MT*I42O7+e>_i&%E?=DL8w`Pl0pEek=hcW# zK;TUn5HHm~(y#OyDp~9834)`F8XDp1dY(6tt=oXqYy+YKq$vdfy>wa|4g%19LPA4f zLF5n_tjm{IMoq;nhwxGx3kg|Wq}U|39TQWel@8llDMZl_VYNvCJZX4H35ahL3Ri?K zP;dfFeBu)KM#3@>&o7;5AC8g04^+UZ=%m3qDOgFWoV$rNsK(wxpN?lA2HIiStoUbNwz@+>IadAU0Gga<6rNh*yRX<&efN%qib^`sAQdzAi=WtI z%TFeQE#htc%ul@z2$uE`wMhcl#BG1hVZI|7jC*pw&hUBP^8Pn2pbi=R@8u2T@b3TV zzeUFJKjIe9=+$$AqW%GsFuJ0;9T5rVg^;{t_(zv1*UWT`bFsuH2l%UbLnC44A5>z- zGCDLqyf-hG{j|lUmdCvr6L7VAk_#}qy{S;&V3GG_qWc$EsYn^gA$a|_pHwg-0~7Fh z?}Rua2|9FM<$_DF>CFZQrm_tdW2=BlK>@VlEe9^f${^xU%Kl`U$ zG_QkY+IZbXNp=h;d+g5g!^%~RZ- z$uu{xdrZ6XD&z0wE+-u!likl7@%HxAI7(B+Oi4!)+-*DEf?=d(B$%n}6IE?7t~X$LH|FyxsY~ zfd+;4e3kn)2z;z=Uq>hhfdJ05;R&FG7(xahA9|BXUV#SP@|dD+=ZPq&q)*MFX~X4I ziMCf<;_!UPEl&*l zSx;Dd`y*LH)JE0ZWXhsy_ot1aD_&@!fYem#A9C4r=4NII!NJhg_4Vk$+r|gA@+OD{ zuU8hL+rSBEKt{We1~>up{IRS5g<`$+K~5bPObmzV+leri9nJu|@`={lFMxDXu zz?UvY(pW)DRf-%>d+OLlmmGn{*()bw1!o5gk3$v0exIAPctQlujYv*CZx}G_b+6d( z>vs15ZDM9>cv++ZVs{>%o3BzmcD!B%r)O~ zaY>}x0H~d|2D6px+xP*P99}O=b;hK@YYXZ;De9cN5iQNzD#nZMenJx;ETB=m*iqT1 zgtidapisNi0&>Gp3MqWhc1i}i*K|E@7E)){*;_};!6eB}p8#)QQ^(ZSGz_#c={NyS z$TR-AQ|qewOsMAH6MN-!bzd@t9=WNnLGvie5Y}O9cOX%#PZV5p6JTkuq0vPg%JNpzNE5`dRU?&om_xw`5 z)Q0m-aK;po%M`IeSv*xlxI|Vsb0{K|k>{88IG4soWJV|>Lr!+cbz_7@X^n{US61eL z43W7XvIFl>Um%BnMKAf4TaMsUG4>M4Z?wVl=RER@KwT0>TU73TLY=HIqY}f4hYXCdB#Y_Cmi5Mr`a z(>c!Aiv)_A3*kvKhRm%unx^cOAYr4Sm{bCUx_zj_e>fPBe;`WDKKJ)iP()A%7(*23 z61@i*3wtmFmb2TM%%~s4BkOU$C(ETa$6|uR_N5;QY0g-P5+j!AOR&A0jfCFD##yNF z{^US39xDsKC3*M?#wv>1mf*s)ZjkV|VY`vu1kW-?=W;CRI`#wT9F2xM8>!h`Qq0~@ ztmx)mq?b@+_Jt zGaMurDO{ak^V}p(FSL#sZI7IBa+2UlcwZ|X!rx!;Y^wWrf?S0!u)cji_%e;JLKwh6 z#7iMyoW$>NK-k~yLEvCx^XH_c-9zb^BynY->v`Do@UQxtjS#2}IH%!mhaj-L!k;?^ zJ$I=a=ip2)+4YV7Qp22g>tjT%a?oSv$Bf&2U~RS3#qs0T(TI4`f5GRtD9Nt>O3VD^ zCu&cJDTlZi?Cpa$fLmjH!xt@aw?f1wRVOs|M6(!BEOcwDA$$(Z8DfOAmi;sM#y!V| zF5lr^{qBrJLvCr;RDk#k$T>R@Ywm7ts3IGpW-il{m}iI)pdsL>#owl`Ph)GRi=)M`Ei~PKslCn`ZVQxab<9Tzw98(#1;|X*%LolkQ>HgUKq;GqR#+Rp zkc7AZA5T-lw}uPBi`u*3oqxZdX>UiuZZ(gt{!aIGRJya1%zrv(ws@SP!iQM%98f^!;<Nrn0HfiwtZ(^;$6ZG!xxMJJbtZkW6i0G4zdQzL{OGBIbx|4ww zvXk8OH1qq&>ujU6w6qswudbW?S3GJ(;iy&y!V!CAQjrH8FPZ!$pa((8>C=f4IFWg zE%EdF)SqTIuGwmY3^eYZJ9SzMd^A)<(GHnYDfs0h-*%Hsp0v zeN-$St9U>ZLUJ<-hLd!4CtVU5v6c+}0k?5ca~+ZPC4KtP`wus|9M8KwJkv9#j-G3C zx?ynY6Ty1Ej;ziI0kimvZTEeBec`dOrxp5mOUh;ie1xs}aXSj22TlN2h7qf>w6l9E z?a$u0YNFBB*>3ye{%O#VMGF^rWU>d<6OV%^UaqGR8axbwzz+YM$S6yzU*7ydx540Iz`;T=Zl2k-?O`icz2;|9&?ClTE$TWhV?aqTdq<8vXyD4K`S{kp z4=#>=--h!fB!|$wto~;ffi&3QBAU`97O}N-ae2np^$VU9{I?eOTvut0TX>8UL$dne z)mKQMc4@`#`Z3RtOMk9M`d8f=8jgJ3TPnHeEJ6L%%CWsHQ*(?N;qEH;WH?=Gkqq9w z3%f@z2PWq4PKeaXL@fU%EI9?m)AhThIa6>2D6d^f$;tbnaW0M@%_zX{>8r`AC!`(P zq-tnJmt0FRd7T@=kPBr8qexD_4ehT~qwfkkj#Wmb${IC)_1#Z_gNCk!1Ky5k#NR7H zm87xdGa%=Ts4vyKhCtCq#4`?%*k)7Ne9;?VFircS@H-HBlh759YMU$2^h)W&iMM0_ zAa)pZah#aR0Yco;vhsf~$^OR&U&2He?(8vQCB;RzAuL3>_gBIidH=5KjZ4bOq!Isp z8h8(Lo?8V&Q{~iEVjfTfXQ#_{QBaglmvs9DZ)Moc91+77Y+u(mvT79IEscN#fWHEA zgi>B<`AsB7)R)otX-mJf%un=)%|vv@e5S$7E25oUek7G{i<;aGbHfHYrHdb3C+L#P znHs=wqNumV>E_;sG_af1k6gGnl0R`Y>S@BfeGD=35pErumi*d*ATd?zwM;QOSpa)H z!Q|*EQ!}@9YdtH@>nyd^%Ky(ds$eYn_%lQ^D=fH=ZT_i_*1!YAnz|B5OJ>z3I&ANE zN)F;}4F78^vX~GFtLAq@e*W|4fW--O8!v~Y58IEynBzaWYC|2B703`~4Q)&_9QD+r zz!f2b(1CSwn`8VYVvtLpz0 zrOcAh@ZQdfnD;=75t#dr=qGZwlwi@a^S=6f9}p2sxSqaE*=!~^{_R?JO7tFTHJ%eW zLm(py2_|TDgtz4E5rhhMBEx}#hF^o=Buh#&l7U;@+VD=bm1>SO#j3C_KH=#RaC5Xd z!PBh&Xsynld+@<8ZJC4By!$HWuJfZt3`6w2hoh`<$=KY1FXWrgTgO$u81W8td~h)=X3j9t-}U;|@D}Kg-U$*im{~v5rM#cs9f) za}yKz+-EJuM~wneB^<(}b#iG{K7H!JDg}IZfE^5iDjh z_@jJHP5t=1vVviqkxGqE!i^zdTpzLJ$%p^>Z*N^A_!P!jrNJ5ak65m)KC`*&xjvnB zNdppt+3{2{#Dn))4H!~JO;((y)902X@#^xCZu$n(#6@cPFsC<&oV8Y!*6tAy+w@o_xVu-4{*`$y8^6B7DN6*9Lw?=`|p76(?_fw{YRe_+?T=1~;AL zck#LvhvVp=&xVF)p1hxV=(>zg7T2^DNWK+J>;+s`NzI&MWbxg2gkzr+B;Vt*-ljPq z{o1`z_N+WWMpeOjE{`myp&U+=ieWWo#ha$xy1p;7c6J=!7FSfg=HAPSl2H`#$GA#J zW220D&Qd|FPYjQX>a$@nWY1*AyTKJ!y@D!_oH`iZz1h9>P*hdbAQ3N7PUEGETT~}L zOmxmjCk&e)w-Z)YQ=?k(*>W+fL#JI?zjO*IhFkW;EuT`)jcS@1ROUl zr_XC3ZFW0QvQnW?u(E&CSS?$}^TLewB2sJkQ0wxr61c_&RlY*aSk?Ds#}zehc%aU> zzi9XcdT!~XQlqzjC?XrVh2YwkV(cg-iaL>c0L+s zp&S~v5dWA>8{tNPz`<%exuMIe7N8h3cytK{C%^>0NTYRFk{a!bFYU^WpxprxW9rhn zEis6w@CQq4_nDn*BYvUa30OX5M5R_EaWV{vy~GrgvmcVy#vFLK{4_!O zK1|sajHUbjIMt@AiW33h(~6j_K0b`@>z~7puZ#HY!z-0B(T%+H8z;p1%4qRjJ+kD$ zp*eJyP*O7~1?wdRx(G6unjiWx%sv}U)4nTsm9&X&P1vrZf_t1^Pc>k7+Dq!w;kRa^ zfHk+mM&_pIo1w+THn|v)ui2}T|9!QC(-CpoI7gz-ow8b!{cXaCo9CUT>X+Ytk8h@A znF&T@uu6WZtH5hdt3oy@X%$=C6v(|N3By}vzi5=*e8X((poGgS~zk=un^pz zt_fFn8W6UnkD3<`sYWjh-qqxWEEw&SiD?%&<$uoW_t#^I1vh z-3@PppOh|`75e52<^&5Sh6L%sV)G5g$2T#YUyO>n9vs$;VEOAP>doCzU~5_OaoO|V z9cN-{a;2uoLlU*x%j0#L~6E#cUWdB_E{|5w%WJMA6Vzv6(zY!weQNM zZ|bmPf3qp5s7Wl;VQXpgICyt@2zUDi$@Lk`8*^laM>Qp+;EvsIFU#O3Gr3sb3q&ugz-)a`@^D~dFD zQKpoa?_9xISXh=6l;;#S#U!b5Dy}ZcbJNjKgRn`r`ysz%pne=zY^R(kc*%K$~*~24b z%l$C2-q9f>x&kq_^8SJJ(iw6-cl8$;uloL^y|=!5yf?q;P<}TO$1S6j(s3ky<7$A! zvT*dexQEJ!kiB3rtK0i`w|o1@a&07@q*wAi_YM2*)`uh zg1w;W(J4hG1NW~Qd+uCo#w;40Ekah_)UVWkE@o2d!v%6hl#J8e{qlbp_}?$M5&N6J zKuYO`W zT20~M1Bn^J2*bnlCB<@JZd4|+k3^x&%AvNSGd^L(AYCzzsdx9xMzBP=^|V0Bu$m@F zZ#X7Z=FgSwo(DyzPp|4+m?gHVR!3FZn8e!0#c3g0t-POq_@ z-si@hXH5qS!!U5veXXb~&fb`0$>!lW@NhFTch@j2R>*9DJW*0AvbbYs41`%Vci8}5 zsF0kT+&v~cKT3AN^#~50rAG9lon`azKjYm=rHy+ZdhIcSxJA9AU*%CP_z%0NOeUA5 zJ~C7kuw5uv=TGlEy7;XKw3ZtylKdH$CuXpzr@DL6<6$&0v4dOq=n0cXmpVxKahfk35pxzw!M5}W+x09qe$a`AL3pC#q6;~Cie>w?39Dp-N z)Kw>r3j-mof<*GdG(yL%V@xR;cQx|>}6(x2= zpD%<&T~_Aw+#&sacsp>SUy(ov8R-e3SB(}lq$%uGF@ns@%#idrOR2iSRT)L!-a50w zAKpdlNBh}K92YNdez~UJal~;1r`U9Z8WSa$RAS$vlRW$$){%r1?>(%)Nc;;XQoz;w zE=K7s0c$!K_BSXidhx(~pn70fL|#?1YY(EvuZ{Eg?_gRC<+5%6 z9buXE(SmUXH&;3|vilUs#yNBh6qHzfkNOT?8hIERCG>wbl17kd%_1u|)5mhu*6IaX zAJQ^9pJpStxSkG#May|8MNEyZu4_ttn^ZNcEaZw&LDIZ>=N=EU#LRd}uTFNODN{qe z`f2uU7f%iE^FM!jtM+1euV>9MlS2jGYZJ<0GcNbPdAqjN+Ib6U5qf=#`;u3FTw`1lR3^k*T1( zvAo==a@0}0sS)Rlw|}d!Cxk9pDi8!@KeH-3?rFi z%cX{`{7%Gaz%tNvByxY93gbQR5A(TS;7GAw8m>62F?`)UyF6SaaIofOkTCgZz=j)q zP}kVrwir81qJYOqF3~x4b^sz1@ZNS1`cimbct%1-NUO-KpNi6|e*gY&a2^a2o|wqe znf#zGC`YTB(0%>}VHbxQ^JUM=Yp`c3qwy5R89RdKTPz`DeG8AMoGSAnC zIc@8>+<}u~R}bIU+QrOhFL7jP!;8t0XsS8sA4d+>1UVMR9r%G}iQQA*fchnQ3l0QlI z#A3Re9eA_@)ex@pneYJ zNR6Yw_dBVC=ik&?KxXK79Pe`ND3$C=a7+Q~l8UOKKYT>wB4`4z%%%((x z>L5|A74zg1GXFlQI|X0gF6L}L)*-pdp>TTXj)sbWxlfEA@;ULn_jr6DlR4Fhw{DW* zaFeqo#RFqj_E92ZBXK;szCpMSXW(WK*zJKrN)`~v4>e^fD8d|*uQ05aj#u`*INo-7 z_HXavSNbM8X4hlV32rJd6o#$aF%>V1T2#D_nnZeN-C{O@pWmA%BmQ$w+%X{rQ7-1F zW7Ohs;a4f5g37|+WtZx4h1G1OSarFA676xYuuf%6r>M4Q&aV_VNA#4^q4*DV@?|?#zarAtk#1Bd+Tv$MD+r!Tof(s`$LZ> zrN7d{iFN2?2uuTsT`YfuUX79he8_d>!(ci%rX36vP&5)>)hidS)SS%0k0JerJj&hW zR$687Nz<4RcTy%-Zj7qP2xi3y5}NuxzohpN>(T*J7`pn`!dJV7Xe>lIlu$nW?|8Xt zf74H<__BY1F~;8*k~{qgSIdS8Jt)Z+x0H8mLYm>gLw+%{VB7fz?5lzHmyPYOwvuy`gi@2+?&XN@xxIC* z=Q*7=A-BGtTS3*QHG7Ncf4E7B z=uOTdRpIY~^4~&3ppw+st>9B5G@Sp4l|4eyg|QXr0fvK@MCiO=!U?JBD!MdpyVd z9F?Ps?s2K^gp=Y6Se`BbqC+p_^K?K2T3SYRJecU@8&82$8=IjM<3ShoewU4{7E8n< zuAAMeZm8x(?r4@?IG<_p&}dv~hF%%(S$(HIKR1v6+Tj}64?cr9`@%-1=Bz_oRdxIF zDhv^aTUFH*IE4TuB2LJFp7oT?JOeA6OMFCo@rPOyZXV(AEJ7q|AVi&cFRm^_GI~0p zbx&5V+zKDCZ^LCu%_*BuQo16Tz8%v?*6S-!!6pf`N!Jdp9z>D8^KbFdVktFJg#4PG zKcl3hqdR}--wm_H#u&INWE~X;6M}1{FPZUcTj$f;m+)1;TbD+B1cCLJd8Y<{w^u~X z!XfYw&A++(A(fDs06A^a0ePojCf%ec9A_RJ?+La^(@41=czk&3+TyF9dt<4}`b>77 z?5-+yE_yyOv@MGVXN!MBKn2yy< zZayteo7cZaJHVF7Tcg_nhLmbY(&9eM=cWRH&sm6|vFf>-!{^N=#v{Ok)xiHKSFA_f zt?2tI*z?L*vRLy#(D*A)94hk{{ueFkSf6w*D>S#OqYg>@6`j^D{(Ot!tiB)y&c5<= zg}IFlkT~k3XcyQ>`b;LqGmBOoX}?Zq_boLwT?Wi~slt`!b+ztYJv^~b&FvwhO{3Xn zesP*=z*ktVEmUcx`ksL~Y?^o}|I}cut<$>BzwIg$E%5TaAL-{_LJk9?Xp{Sy-tQXJ zujm`7HYw=;Jpq@@bc-)Kup@PU##XbcygZqVX{C_*!-g#*M3c@^*qzuUO zRRSjp)$Wi8`pLQ6M5_d7AkW--b;fJM()VMeyCC|s8TQl$Bmru>&Rtf?d?+et*gU`sVb1eBoYcomPAb2W(r%yHBwl+qSemL$me}1@jcjr zI-h-uYRMc)(a^BMpfqsXsVTue%c!laxx94xH8vK)#7LO1Y>lk;l#%rZcU!iioK7_+ z*6+o(&WumIaA|Sn_i1G0m3M1*_NR^^*Cl7x#I~KzKw|%WphNie-AgyG*5Y<-UV3bl zz7O4xD9i>EOdh9g2mNw9pE*r}o`Wf=SXC z6=FXjf)>QJ|B@|C^hCFBKJ4$LuWXGH6{wJ%sS-q!|SV z#r&Wp6WEb@pbYLM2Sh`h<3zTh7k| zlYeG{f}hsxI6;Ka>0rWd3(Mpy3+;+xAc4@~W+*ZNIL8A`i`O{xK-A=G<)oT{PL_wb z!r~$6^wgcXl2O+5=EjbL;^KZpMqgn$9SP&+M;jF|s?DkQ8NnGfl5e4rD?~~nvfz8R zkJXIYw+i!EQZhN4WNL>HV|HcZj6+pUyx@m5*n$|UK?hg55;SyG=%-_2()*bWq3%x&THL(-=vCmU z;ytWdbi}jLk$zxX@f&P-9uR414+vxFlclLpfECDI2_dHx38dJ_$VkxYWn;}{Quqo` zM{0zfI%P$0vtP=WBcVd@i|vR`h#K2`XV|?|lHF77aDL?dbLFmXbAZ~nd7T=oahRxEXLF<* zhlz7$132a>YgtgK;67hnM>7sRf}Txzr6h)+D4h@;AuYzs@Ex>j z@n!>A1BQ)Bab#6rJ;lvo(jlR!y>By2jrgBo_N$+X_{RKA7Vn5-V6nV>7k@uuDZ{eR z@QAg_7s3@`+!JK$1pW8;#tg0-wWed(v4kHuor$s%<)oogtxlxx9(l%nX2TiA1HEvG z7;DT@Z!HY=x(knI!zo86B=>(KNw^KHsIhe*9^G(H-2KB0Az;j)iN=2P`rYdL#ez+oW|0C`x zHMr%;jpT=B4W3tK;6R~s*mjFw-^2NERbfiB2}9!1jE`D2v8+EUXpk~uY3V424PdP| zMgGQGtg}=v`x&$`X`33$Blh((R*-mj-X;m!WFDs2c01CrF(!jsdHJtMEP4$vL3W@^ z@FBQe8Wd3s$gRXl0vrYnK`?%MjvNA`iAymLS)rb%NAYjQ?4Q6)4j)}6IKV!ty!@EH z?60D4jN~ARg6Gv$6&KrYWX9|-p@x>a62y4y?lHL%$Dfuk3$a*E}wuW$-^js#R%ybhU;j5*jwPW&nCjsSM|{Uku~X&41f4d*6?+ zNdH@b{fS)CTajy1TqdSdXTK7hKaLllZ)p0j7HLhryj*FM@YORlX3F$>noyShsW?^d zuZ|fP1aFWHHIu{&L>KWR#{b(3V9QNX|M>4u zM_~7$ds;bhRiW5cMSO4ORKd=+Je~Dj+f1Ouz#$-s4UDfrcky)I)uC}Dt4!Sn^qD3q*l^jQPoh=PqS%5 z&n3tYz9z88)?J{@`Sp<{&0B3x3!g}sUUN6&$f1E>?b%*_!DP~TpLcj7V!)uv938;v zepOfgw#>kF{~9f5uUBsL6+E%$J@^u4rUCva-LLdJ5kKX#G4SpcKVV3zLOh2pvxshC zp+)rS>DG&Af3X~wFl-zB;45;q^c9%-^?b~YZTRlfW(?DsO*ENJyGuv$-ZVco7x_1U zHqAaOg7QO1-C|~u^rARpNGGrc*#5znWK@sF3QaK*oZ_vMbEpcI>1Xp6_U0WQH05CA zdgO=+ZGR@_vQMU`ZOlGvCpYf-OB#;;ESWx0w|>Y@;>V?mnA$QwE84JCbFYPBcIHIKShOLwB6N*_GsgJ zJ6o>4D8S6g9h+tzy*qI{5>mxxnhp7NG1fpUvgB>qhDndXL3Fesob@M6+u`9BaH&sbd-tOIKKU)=g+wZqgk#+;|0py7`$|(krtnMrmpU z;33-Ezpv8=XWbH`DMbw~$C4K>N`DN7NS)UZD$=xKdKoP9<;Q?Ebf;|DGp3=|nOg5A zTwn7*l!j$VppqiqeuykB99_cVi* z05&zfO;#W4!B_NU9yTxBY&&HUTv}o2kF!nkwqtp3=IZZ*&d09bhvgdNvt2id88=5H z)nX8oGQCo%nA6WL^ltHF74d_-?K2uZ&mpmh#bfn_l+5c!%#a)1jdGbQkNqNM)`O_v zr8dCt4J2jzM6JSf%zGE>)`aEJI=*S}*g=o(nGY*={lGv zx7j%n6FreOyUM{kbnfOssh3f5lJLEnvq(q?@#&&Q;pmqSQy}cm>%1&pa#hiep0gZ1 z-$*m(*OzPF)zs@+08|CW|a~R7jnQ zS<9KczFu);pF5p4xNXVRM#Ys1^QqpFCee0YT|98OeCkniB`-dhwgIB$@~t(ml_P;& zAT9;~#g8t0FOZL}V!;87sF^;dd(41l(&H8Kbwq_;rtmK+x$U;f zC;j9y`Pz@SYc3Okp|V(Lr(QtOm6W=BhC$paA!**-V&J3JXhYH)LVk1tlcK|bSn6|d zbwZ1D#rHO&n7QLHQbD|T?bJDQlYXLS6Srv0lSyRwZ*i!2If=$q!qmoL`}&;AL=cBX z7+=cVcApay|CPP`uT5PtZp@+vn8}gN>PC1G`Gq*O<-PGT) z(>b8Swknk??e=J{Di}=4O^iFK?R^kE;T#)0;p&5uf}Z&5(RTZe)`((uNjC9=N|-Cn z22@Yr(^K=<+A3&$>@nh-QyXVMVb{VXFBQb9K+J{OLO%q!Gd7RhS_c3VvW`oiix1*3 zHKX;+NUogc87d6&K;~)jJLF z6w2?*$`NMWvTK$mC%b@`OCLZ(bmUcNRmHwFkBV>~o8HG`5wWLttUU_61!;7p8g8GQ z;_5naj-!Y7Bv19arQ7=Xos2v|a2|JUb(NhTcVRUZFfxXCJRBRI=4Tid{rLOD4Tj!@ z08Cf6F*x{1$1pN}M%o8jqMX2i8C8&VaUWJ^cQ{gp5B%R!EdBHzb_6cLVKcK%SB zg$S-{uicY7DxT6uTWlz&INQzNbTRQ+ODP>2rze~7%4a}KlK88zfP@CU{4R)4?ph`w z4)y>+N%xc2MkFufuy!H9{-!ms$m3~(6q{kzOS;%iSH>d`P|J${$ zu6UX!!yg)F4SbIz6AIcsnL0zZDnlcPe*UR1by%b-E7<~EfwjZBm7~OCWqJ9H>~Izn zFdd(pn+qsI%1TNY2;ZTT2P9=>k-@yjDwqMzY-usrH(i9&58nSzY?;&CEq0cW++VuVcO{(uuuMa8|m1nufb>J}mk z3&CK?7XebOBK+`fFPAbTYKXgeF7JmCYr0TxG#^=s-Y{(m{kShpe*V{v7j_8_m18qZ0( z=Bf$F0IXnuxo%Mm%wqyf_0Bg`77BKD?DSH5ry32-Pxi9-*R9nlVbH(4<1@@QDjHkc z`@R)(3K)KU=l_nvzjH0Kq$Dyb3PVG)+lVc5Voyd!hJu9!0}SqF)6<7J&CeXKwtpX< ztFrjG|1ZFXOBt#)aeTCKIxF|O0CpQk&qFk7%6cnKo&_28SsgDh*PSGM8V?(Ji)Z+) z%o?Vf@Am+{j~{sH0U%ls*oNd57e@e;;7%?>{g8Pl_`aFzjgFlgV+nzQh30hiLVnG9g)y}gCNKs4w#i~IRg0nqVy(heb0k2MY%EWpg5 z=MHR2OK?5kPg52l%Z)`DJ-`jr7`Ux7nS(*b4GDdr^}iW%5k}5m z($D8|h%7R?==P(tOvvF z7Bm=E{jO)~rnlA$#t*Q?1qmn*ciUEjKh$KZOPc?E5tyH!uO@g;ND;1n$;h!;2Bwhs zB-j|%R;bIcxyHy?S?OZPO>h!Mg06t6)|v3O0=1!hGrDg*b+$c$W7W)*P*#BQ8+j1 z&3Y^q+d&J2W{u(jvq(pf{--YOh{cD<^S%+eP$csFo=!ffgXB00=0llY_%0khZc&jq z%-4Kv0QM~o7y+{+AaZa`PgNcF&DGV$+tQM?oU8R}VtbY!D5>C|B+Pskc*RkAs&<*WB8$**>DOQ!xv9+P?(n4#}RwjmWl zLj2msdZX-FcUT>D`fP7o{B?nyvGONsLp!06#^&^k1b35SV3sqV|LL{}3kmMz_Hhz8 z1%di#WFRGk;`VC6zRGFqZ+QmB18=(o8CX$6@=}cO`cJdRNJ&X&4_!e&2cdc5?iZB3 ze%B;Brk+E<5v~P$#rEToPumiQ+h{Dw;dH;;zX2BVzv`Banbs$6(fhd(wzO_ao2g!1 z$nmYRwohgfMs!AyE9U++`6s_q>|yeFRy3FJ@S4&2*})^E(v@-CK`G&DUmG4r_BF?Y z5)8W5#{p@kITyGO`4wvycVM$Q5L|Snre*MbW*-_a0)h0a|EsglHmLQYnd|59C+t(O9f~jSJRd@zO5arP) zVVzVpCO?4~BO5uLQ*r|qF$8O+l@#fj3cPmA(I}9jeEGZljexJT+k8PsA@qjW;`y|IEb?IKI=;6Wt!tQn7%?b{PrQy}%_z}&Upfd6A-}&B+@CW79 zF#Zkh61t3$-ZcA`elXPRWqe<)&h7!W=+x0dCUVMzGlLHG%|g#*TgCH+Ik&{*wgQ9r zRg6D4%pGQ1DC_@p=_>LqjgdDB({R3@O_60dlrXbux)_(o z!^cNniTP!_LpP(acOLlLe(3dI-d1+aS#4hT??p^Hl>sXc+q@7xVL$)DbioggJEG5j zNZ3sRf?V>>j@E~oTGvJyL~Z(m9@85`V6jZAIO^Dh{~?AJiWN#Vmgn@Q{;@qshkTJl ze#}b$nWc`$@;bfy`EllQ(e*My9pwrrhyaNr0+{~Df=sJc-|*Q$buA|d>V)vyWnlEg z>}(&PsJkh$H+|VA+K&2rP({w2&-T5sh5v))oDcIX(1$jo1dnzZg>{66GPpv)LlOo; z_eWxG>x6A4yTjV=O(*?Vc9-84H8cVZ%U3Nq{`H5A9$g9JeAbA=?dlyHTg!fA;u;v; z$IC-g+WyhVx>b>(FK_(bJh{a|Q**Au6cL1fIzd;Y0k(rTyJQJ6*D{liNbdMIm!Tv8 z+ht`3SwjF%KmHX$_*|c|-LLK>I)bt3f$rQMv+nZ+asxTcpbxU9si^)aAW* z2clM)HXdVF3i=?!;A$a%&JQk6Hm$1DgyZx3M*wJ#JqGjpRH8g^=Xhi>TD5qGmMwJ8 zDBj6Nb(gk)EX+J6qJ6cc7?c4 z3nZ{(|!39l3&`~5u@<4Y-AnNU&vY(u@bJg^~GK@%4T?;tC zJT!NB$a7aDpvH;tE`7mWxtp%*5QoAKe7mD$bpJKSyQ1oBkhRjFS6s9yrlWmc%fZJ- z7$9EAJ1R+m??*dGAv$6x#fb!6nE;m&oZls%Vsp!%NNIzoZ%dj(OTN)csyEAt@qq*Y zoNB=>R9S{p5MxF-;$6uZciS$Rqg~ySkmu?aE&4H^-t0{`o1+SfDav@?h^(oM5{hHR zG}#CHVjzKtTrWBYfdHIclz$GrD$@4W-U}$m!>Di@wdYEUH-M?@{B*mcuA+i0`e06NECg%#rW4(ABJI(`gS0eVCZBF{S9;STtleJDiBe zmI(L_M1;LhB>%iR_JOc!%VA`_@VLbZCSD4YKzRa|QAzn?hC=mS)scde7?Yci4Vfs! zj^L9tNr8=J&)iye%Wv4=0k0rZgYG4tjO>&3=F5n0s}tI zLR^^9wTp7kh4)WqN1laVSUm+WGm6ArB_9-~20PR>JtyHsob;&$6lgyiRvP3}_n4bs zj50}K4hi@ktl8oI{$P_-Q24K?uu$L(#GARZ^>Eg_M!nKfktO7ZpGJsuvgFj9R3l_w z{_@B?@_fU2ifJiq#xTHq1~1Ia(@E(2__}s;5n5TR#naG!KF0RZ#~tK}cFGi@Q^1RI z1Vd)*CS6DXyY#8$cF5kOZec{G{NucUc56;R{<8M4&A`lJ;91)yPdZEgCd_d5#2!+!gCWOw&YuLOmb z?kbCDcbrGuSjN!~VS$?!{7V3W8WFz!$=2P{$W>jft~t>blpcI*eQ%60S)HZsX;A)H z={WbW?v?zrl03Vx(A|5i6^q!O-Gn{u&d;Gi571C>p6N2Y*Zt1==}bvpy~1ha&+~(B zim+XM3$*XbSMH=YYoR2-$E6#P|9Zhn%?n7B@Es*=Hsfr+`U#q;o~1eX@@9O5-ZQi6|>$nsEPsPI!L-V0nbK@fJxiv z!n3)@&Q}7pJ!;<`XHYgqW{fQ6#vk!(xc>vLThsH?{R`n1;K4D&yQ%PwV#O=nuV#2F zE%&sRKWa*pbUF)t9L=3kO|jY$wM!ZHFh%9^X@>$U&6->5Jt^=RL@)w)<`jkb>0cw{ znSW~1$HN3LUYD9bE;IhMPPKBn?RfZ`Za zx^upEeQ#xHnOR+(0OYSK_fr!v9xMx}$jYqxI(;YFl+3n334=+bEL2Xt4rV>^7uM*+LpFB!2I6g)|lim<#|OEm-ctG>5Jm?YwkoSBX3E3zGSc%<_&DA+^onEboEOV17fxE8s#p znk8NVoE$DaBc!MS(dA}vcb`hdx0O6&M4AGtZU0hDM@QDhg&R}{Y?m4mL0gFP&51FH z^(XhrZzskPo5-C)HTpm5gk9Bl3BnP+X?4pE1eMnRXjVTI76CRewWEYSCY4Cq+T`_w z=gDut4iy5h{w9!Q(|s;0EuHpOwlnAWFhUkdc4K|4DzIqq#iIP*7G^>20<)fQjeodB zgvm|jAhhbAW8k$Mfo}&~DRxeK$wmK=4DOEQ5DU4pfgLL?1H+#C7(3WBgJixpiZrLG z=_g3-WwQZ4)g_< zp`SNxYHvsNn~xU=Nul@C^0U`|TnApOSvvDB6fyC%!zv|ZWjh0`%626X&{J}CwB%Lm zw=VJxHL?sA{cJih+lNJ6d6X4GgmjMnuujLs8@BLAw7V-9oPS^N6 zbmQY{FCHo2D`V{*V}KD3xumK);9QLre)r+Aw6=ahYb^rSNp)?l)0F0iQ01R;b#LLN z7=h?MOXR2)DFrWFx4X6s>-~DGaCg9r>$`SMvBmaIEi9HI*f!zLIeCz;!%R=e=!0{w!c{)tKXD>v}G z96^+W=qsF=XOStc*)69E&r9nU7-ss29R-oWjpQz^--X7^QzxeACTK)5>4_lL13pk- zmyisA*nl0tPK1`9espww@7TBmS}ZaY(|As&KiXH=Ej3I;l0;dL$87|}@{&bM;|0w{ zxxZII*O>+tpEzHfBMWi$xgGxlgPXy}HcWr?tY^QlO#2I8ir+=Np!hvsW24jP%1j#{ zdU5fkCz^Z&7;H*vX%PS~%FV4UY+T&$z!lt}X?OqHH=PlvoD!0f@E~@Wf(||=gQgUq zj{rCE;l1$mG;`_A5~q5-?9mniGrVAs2T9;S5nvai+z>gSI|$$hF1Ep}y21JU=#5+& z{l13!ZEDUACQkX%7;IPn{*?n=sDR+7+;J8Cnm;ZgB4UGm3g~P=OW}-qd*#VJ-Gc`& zo)8R8U`B(){2A!yFG8}t9XfKIL|8427z2VG{5DK0^tQ{n=dJ&nHr%%P7@2TBs0TQA z4w^+?n0US+^O2X5hW>VAzpfFULmaFy7FMGX%hI)TZA`T-v?(N%XHn#k;1|Wx`|4% zT{v1;j9uMUbL1r!5P<2CloT?_sJAp)N%x}ih|-ZAV87fsE=&c&(4P8!YoJJ& ztx6;PYKcU}Z(%yhD=VM6{EtdC>zF7|YUr@A6zl7Bmis}7y-3)1aGb!a@to?Fyq4XVvG`HR& zi*|J?)y#RjdU2;kj101l9X^VRirGUPi(stgBMxa&?aV%YSS?IS_T)GQ)Q^6XF|e2t zer8D&7k~MG?xb2NU0wm7FJZ3M#qi<$O+Oq<5|D8(s z@5A$3F3AN;Hmu0P^dGOo-?PbE+SuSSOWc8XzkEPqxh7xo!0$aW_OwXV{JBCD3Ya!L5_TcQYJEp2q>dBPd~rQY9ZCJt&ic5Z@I=M@Qwpnd7PM%o5&w;{TH)(Jo63$wvuOmJ9R%cCm3!Vz*-d;SthpyfYWd_B8=OqI+r`J7jv zUX?(EOArpRIDdM)m+aLIIw)p&xynZJ_cUMCH1WaVQO=vgsOHV{Bl8r$%Ju;+=VRL( z{7j&&?@g>+Q6GNhm@jJDTnNkGe^uMG3n5j3;kBUqc^i?@K?Vag6z{VvO}J}jV|K~J z9y>{n4N^YMT;S9rICy^GE4s`4#!36XvrHt2{jCpr)x9g0GkfjWrH0!7yfZM*b})2y z<_yLNB%U&?vx@)km2pzBzkLo;AWyCRmK6Qw5rwd?1W&ZzY@&dc>J!22Az4K=%AtLp3^ey$b}OAWF>VBH{nkwnE$^K#Wx>E?k{or=su z+<#TjANG@tNgV>rh|&S?Kl|LAurjLlB^II}g&#W$q^Bmi`bvpuXl%3Xxon#MM!DA1 z&{(|=o(d?GQh6u;-vhxFF#QR1xSn()U!@etl*@)!#bms}xL#;y*J!tTJ==v^#fv2UzCP@i7vecc&}adS9cbWVK(5&Q z-6Q$%UXfIgNK%Ead(GhlRO{t0l7^hrhWN9$7zD4^sjbpgFa z)6M2-ndGois^4g7XGf6!UH|{T0h$-4^$PTg1Xj#L?L?ABFo+3O6}1gjNn@F!k1Sr? zKNcId8ral2zQcfjIT)^0$(+}FSBDFM_C!e;!;@K7*dA=5TDa$Vk$4SK;70`3)(;Z) z3q+N*Z(c5bh&23Fs4HkxkV_f18{Mxz?Wn*eyd_jwLrjjMe58;USdUdB+t7PT8Q7X?<1HQ!3?{aD>nD_-ax}Cew-nq9JXD>TuJYI!zo2=ZTP2< zF~jePAC-^=uh|E-T~a1g0=HQRJ`U*(Hbl2v9~gKQwSe;8Jf5WNUK8h=F{LIQCS=z> z&tfbY!a>?;uHp#!6a8R2JPeSptc&Omh$@V%jW17(t|~Iip#IxbY%MLV?cDr)KX#XY zlR^}==f2+!M!I5eH|Z8;y$XVO>W(9r?KR=km5jygsOknuyMji)zL}jetW?+yoSl3R35slv6?YYy!2LVa-v*ui5Gex80hoZ{$cT=iB}s`;z9z9 zx*{ZABGu(08q+(|$2Y(g9#M(%^*(g18z)Yaqx6@pq#O zkk&I)L=4EAnE5h(cbcH@lci+o3F}_0Do6n3&vS6C&wwqlb#pEm1aFpwwMv4Dm_EgS zMEnS6Z6ySo8Uvq@u(hnPQ27wSAj9y`p<$}WJXlvtvy6d05(0x~Khu!Fzn9RCD zuv3Q2K_OEhorTGZ9SiC8^SQgwYZ331fB#-~07KxhoNM(H1y%l~#}dZ&>Wh2-Xz?nz zQ1H{-{3(vPi1C0Kv$({oQ=$cwBw}_GIFNdq0*nNpPJ>S^EW|;AD<~=X4S}S96iQW1 zO-x-K7x*9P%ChULA~YdApj##-D^SJCtO^<7x0}dY!#%(w-Ie$u=k^Uy$I8&Gbzv1p zeVC6nSwitf(!-?b1B-(UO+GC#B-m(%#E(3@8q<<$H@^k9@6E1FvdrCUkfoq;nIKP} z-LSB6n%$-jpKva{6Kt34=5BK`B3(rirl0{K6-Sm8OL_EI7?}t$f@asI7J3`E>tVlK zh^~D(0VI}2S5S@q-{Azi2C|i2Ufr)lOI(h75%^IF>oaRRj|DTF&!mV)Aq`Hb#LMCQ gAN~{fwf`CM+gBzYX8V~=DDX#CQc0pz%;?+y0Ci4?S^xk5 literal 0 HcmV?d00001 From 0b8c38aebe6ca02853b5913cb175df9cb4f78501 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 14:50:06 -0800 Subject: [PATCH 009/551] smaller --- pgcat2.png | Bin 48254 -> 44624 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/pgcat2.png b/pgcat2.png index c218b0b3d3600073b941ba6ae4ed6369daf21a5a..c24c6a112076e60338c8138dc8e9c7864a615cd1 100644 GIT binary patch delta 41338 zcmcedRa9JEkcDyg;4Tem2<{f#-QC^Yt#M6Aa0~A4?(XhRa3^?xkj(ub=5b!97rl_x zxBEz)s=dFu{QV6xw*>=<)Gg5;2Q-|9v)IeaD2@RgM=M=${q)K>ilxbfEXx{JppC`qp3i ze0yIC3B(Kf({q1&A%Eb0=>F$VUdrsrudBCR=7&c-Cqr!cP@~vdJR+(kX^1)!}3UZyHWA z^VT(B7Be2-I6+`Fx&LMrx7<^4(@$_fBAMtJZ_?uyiA?l zH#19Xh4Bv}75W>8%-8hzz7$b>+n9h-VvD~q9cvs)7nLmvCxEMx3TsR%6hXc<-61jL zd!r-1E&Yg?Q|8%9S8?RC+T=(p*)HPLm_$6SmYaB5LRB*Rqq=%wq*OfL(GU5W_M;r> z7}^vAxpK)oUOPA{`F^V;X&TUC=D?ZuR3PZRj58lQn3{0L7w<0Uo!B-nu6u zSs??WJ`=#xK}`}jHL;E{K}e+VQK#pp{`HSQ_KK-@ZC`h;_wPJ!r5WyW+q+X+x%W?% zRgR{=UJDdG^Fz;Rwnt}`Nme4qu9b7QjoXkF#1;53Sl%&Y~Atl4;uH-Vzd7V=2-@PJP?1Pm+Qt%ZS!xN3%k{QtZ(P zp<(rw?R-d{mz2i{E{2!#ggIM zpn&t?xJ{8&!Aabv_ST>3Qlen~57w0}#6*6lzJ}A8&;v9}xZx$BoFcvvVSi)^OMJ}m z^1Hrrzpk8`*lkhY_U8M^Mz>fSIv80vAW@Lx z;RSmVlCRh^RxhiqXL)}3E@f&^Cpz66IT+972rSDv$`1Zjexvd5tQm(D+;_5~&UUV| z?tS7nD_`t)VtCuj!;QdA3Bj|Pu+V!2f{#_uVEKq<4E&CdiD_eqC|F-$P4338TLuTA zr0nlJ9A7_cbCJ`zvE{Cief=`e3DKz43mN$F!i7k%S(sB(tL@h51=d#<&=%A>1wDZZ}Gjp>kK+>XfM<%6Ul@%C8PZO97yo+DY) z&c&gm+BG_hQ6y$oKfzZ`g-?BlbLZ#0)0ldxv$IflVU0C_?9H& z%RiCJQ;;=&IUVG-pyw38p)r49v{?PYLZBRBUypH4-3GH8uE{*?5>Ott66TXvNH-+; zOlLQiF4rIC9FeOg>uCL(cmByFDh14(eIM`VE-hVIX^ z2k{+&%+DZH1}G&#I#+Wxs3Y<6nSC%aj-a)`W{5`TlC0x{jg6a1KZZ$J$BgOY0rKRN zMCRx>H-=n9<}zp($PL@aOXBU4jpJW0KSUor`MqnDG|IvnWLxtGT)QE%3rUl7pG-HL zej5{xelX0!TB~-7nDiUbQ(4>Qw9Qz;`rZ=n0M{VfC3a-IG)za4Mk2x}L)UzT zPwl540E~&E=p@JrO7Wp9viwl$TJEj3Y1Iq7xikB*W zEAoS1W11}}VH`u>`MzC2=p~a8<6y-;bVsaa7Y4@)c!-#pQ>l$4&B+iFz+8o366f`) z+-e-Cs!M8^np73;o`<{m@JwEF{uMc2YN^I=Exb&;$ztxo?@Q+A`5k64Mae@HauVPB z0gG0et#Vu9gG@Gu5}Z!{q5syDXTeKaL+n6q9!k??JLZv-gv@{pqyhFJ)YM1w$WaB& zc`G841y3KpqO-HAN!!`G(oXqRZ<5X!yAD2BRDUi@r4UVhYe z*usZ%i7hY60!r34(bB?RtuUn`u_d9jI>$Nz!y1gfJX3ZiIJo4aR2Oum{eHQU;eMnC zcOTbc@h=~h@h;)kAu407x_sx)^YbUEQcS*jd4Z%wkyGS*;VX50>=6R1IknN~=+Gq% zZ4+u<#=p!*%?$TUNZ=zE;x+3od}#e(7`PaO@q4jLO$+7(l9A~Qh$Cb3{_3b+<`PS% z#fHfCWvZx)pSFYhx$lSfgAdWs!{iGc&k`dL^P&WrR2eeFAXJoS2`wTvY8Q7^_NyUd zm8+sPG9T@Rhs6Ir$TXMm(LCXq36BL|aim+aJrj|jRufraa{4Mt<9z3#W6d%+oDHS- z$+-{C8Fzs)X@GvKR0H?lG+PYOW=4}1*U);A!&5ciO*;)@H`zQzZ5=^ZkF*u}_^*As zxK36GbY}AIl`MQXrJeb#4}=*8Z`8A4nvm{zT!$y-Q1VzU`&df0 zEs%;tAE&@Jo`%4svXO2j7^x6foR*jHxC$~yOa)2VAK)+1a zqhb>yKZRKThNTWRvHO7XVV-(cE0jpgNVWP~3E@JQ36_|pGs}7`uIivj`i#a?*<)tg zTM6-VTd*XvJ+@{5SSZC)%$WS0LaLwAfRjk(3s~}>G%w8LPnnoVJy4mlWyReb*sj1A z+d9y;%@T$Tw52FWS1I1q$6vm1J!ZgfpL(i;dI!12-sVsK44>P_dZ+jnG0(Dq!3fXTtkP`XFpC|Q2 z#JmiO{QF<~P7GGO*Ll#Iuy`(rWvr)>Z(;Cjh2O9uF_WsM*x~#n)Tv*wa>CncQj%yG z_og5yHhYm0aF>W?thO5$M-n!MT{KX#zH|B=bg_a3kI;s!U-xk0eaJ@eb5eMINz6J7 z`Ww~=3rU(#5y=ir6{voCZ_%uX*0RC#OEZbL;*;iAI=#KBOy7JG%?u z_OAJ^afpg<7lS7oghpiPiF=Nq{WV#H#f%Qz8txDph}~guc&WtLxy5j#i#(Y4*P2YT zOX{0wbe>Aow#HZUpDvxpKSyY={FV>}w_tu7&- zV!sk9#%y3mHPz=q>2URh@h;o$sD7sky78b)*OmKtWJ3Fe2{F@l!A=H5XkTifwU{TIIdq7+srcNzVd|*Py{uK2ao@ zX`0sdPXqAwGQtDxUI-15UYW7|^Y_1uAq&YqO|jrB#96Tlmjdea@B)~%(o2S~itDfi z7r~~uDL-QrG|FS+vP} zh9i(cNd-;DN4wJJk;Kx8FFB7Bg@xFQaOB3fd`{T<1!qGT5ovMX(#6#U%>q#+JAH)b zD7`*Rte+O0!LE!~8EA)>gGZhwo}<;F3ttdzic?>dYEdlLXyPk6sISYL^9*LWCaH%; z#HwX*q*YN|!kjZ3lN^+xGUC)Gl6oo^6%RK_usl7pxkBGtMS|ld9V;}mXuoophmUpz z`vwKu^qRtcA6_I+{*>1$Bm}{^O=W0vX{8C!6vM~Me~Rys!#(i#`T9+U%n1Y09%|Ku zV)u5(Qz;D- zeq7VQI!|9-KZ659i)3z7KH^K_(2AYUG=X%a9<0Ob`W|E$aI#h6QJb*;M36%vaDJo; z$a5S*FoQy+;zNz*BAm{_DU2^iJ{2b+O0iy3nIVskc9~PkjJ2Dxs$lM;)51j%L#zD2 z|7lxCjCrn3*al`=Zm$tC%V=6&o%jhJ-3OJf(gMX43KP39R8#7A|FegV`W8~*!(3M; z@xduQCixr4_nu_LRfvwUWC@WC96~$q&W=L-bD_+GUYm4|X_)0)0$O^1IO(1RKl4kG zz9+P?(l&mK`nk?cf4dMxXgS`MOMAQO4Z`Oz?=p`sf+gx<`ztw++A{IZi-zK4x(UD8 zcZ5+QZ{=AZ+_U_M?+NmkSiR1-)N80chOj~pGF9##txEPdU=18Nq+5?)P@IxY$GdFQ zyF!o=ADXrV6&*%W ztk|IyMd>1FUq+Ed2)`pqa#cwbkYTK@(*xc9vgHHBn{+_d5SvD*L3CHOa^Cy*u4*R zr_{iMDWaB@yx|+OKu2t$riDbacv^$vjxT{|n%E0>jg6n~gDO}dMNtG9u=(I-Y(zU7 zR{KI{QFVs|&~DI_{irp=XIHo`YmewP{-oE#>w%?nT|Ik~#xj~7hLMj{nrx@$w zbNd7@vGYQ&1=>fX=S=SG3IxXso08*>mONiH2o|Gn@`RP4E?lGkjcW*r3n>cBcv2ol zL4Xz!j#M4I50ZWo#-x)diN{6gMBi(bEDC!o?BE^>`R}Rx-1=}HypoV$!VPj=6y7{d zuQ9&j{k@t@oMzCAra*2vj$OP}QH`{66Qgjf8H^apftOZKv_gBGRS9@`S(6~_&@Ul1Icm@s`X;QD4 zsj_K((XpWOJUPWylP_V}lrM=Se|Pm)goC^wx3Hmu!!G^kp925H#F7w(EsMq*Ly|c~ zOzaQmz=Kr>^A3JC!m#GbkGE2SSS3mVNWpDWbeK`2^<|$)r(RqY zn;S3!YGnhZ45Y)ExH`1nzQeWGeXn3O=}rH+c~OZ{mUVidd5em%OwoSg6IHups+q(k zZW+?;>|Yx9;dEVUYha%68wy==H6)R@&F5*NfHW4PafwSwiI_2>$eoD(;#VY{KKPR6 z8GdttA~-3Onf+pO!z|`-$R2BXBC%6?`2xDwy0t~OnPiG<-*lUH0R5o+~44f+SZ z4FL4q1S=BqnU^0zTO^N#Iha+|mhT?_ z>RC@ZW|f({GVl`HPbB)K*HO+gWz0Vyw;(w>6U}SbNL#H3#&`vxIfPGzdxw#n=g0wr zwr}`hn?BsCm)IWL;Afs+g)W5qCHdnWEY3fWwP5^Bn0o%1$U+6^?69PTIVs4J7agsf z!T+>eU=+nzNZNY)SpeaPLG;0;1BpD-38JBdH&i4yEp{L*HP3&XQw6D5K7T)Xnr%k* zqTkCmbh$G7rM0Fo(G(eGhX+kH0mYpul@yN0X!qy22zFnF?u*e^tE^(-Yg^~9|AcPb)16c)S4D@c+LBS!2-kL>jK&WCJ*?k9baPd%vnnP$@llh4rC zK<9VmmLexMST3}0rF4keVps^P^VB^^@6jL?;qCI@KRpRajyhi$JVD%{s9A4tipj@H zV6s!#pL%p;Djy+s;ops}{!vHzt;JImu6~BjaVcp@Q%(L&uofI#6vo4|HNM3uzIx^s z%w)e=XXg=3`h!^N+hx%|8MqSU%CwPmAMur=W?53^0f&5iQ*LM>qYGl)FMVB4Hs!oo za5-6Ch!%O~Wsy1If^+cOim5T%*)baSpd#el_$NEWf3X;l_E)NGD28zf3vPFbe66pq zzXWIlGiSu!9M%79lggJFzJeQ9Qkoq7P&VrDdp*snY*4;{7P0G5Llm>c?hG441ax)# zM<`?7JQ!k&9O0r>I{m0~^AGG~QVa4i)mHR*`MGqP%i)q7{&}6X6PD1D#LrAw()ZEE z(WM~1xtcmbl9-*s#jX+%4Syda__$CR48!Sw&`gN0b~vLkid^_?tw#o|l{|N^&rX^s zu3o&JnW$OEK4j)c*h>Cl%YBcSawXHb=B(Ehv-PW(5X|@!g%D>-vQdlKuJ()jr~cg4 zSh2w_9RX=iQ?+C-*Ef=H(RM(-yLXn@oAzLCTP2WGNl~_<^J=$$7caf^$xvlQ-ya$e`al@XW{nym>Lu{*P2(chFI`PNj9q9OLNe}hbpsMz_pV)hf^ z;AAmnad4>SQGOjEVy{~Rg3Pk5OIPP;UOEO+Z3%}3<798P`W0q?6z2c=Ew5) zW)U@8a4cGqC~otz!Rux_hO>ymIk*{H_?}%uU-En0zot)7BS#(UC5069xoXQo(%llc zAK=`ZQsf8BKNF9r_Fy9_NfxBdsS>YO%K1mD)Dd`7W@TsUdKvkktCx`ZTar*0zJ*(U*8d`|2M@1_ zmRRW}%oEa|`$5~J`Ihs$)8z>c0WyNHul({K$KHn|f-97o67>`h$3?qpMHqVeu$=V0|zQK#>UgKGqi=Wr&q=Vd1;g~ zXzTuhW(dy`)F1Cr^cl>mN9q%8YWo9$mSv@|d!W*_i>_shJ|+Na>eth!gYi$V%nUB# zb^g1zG*|N6<#Cy8r5dQKa60pWY-hjGilwM<$t%<;H3;9gxwbF3V-H5kM9BR74zwYA znJHA9U`a9uQW7qS-+5Gy#22#9w)~lSz8N!SUfFwP^NKI}WDoOIpN3?)P{}o$6|MY7*?lykaMtE;A?PeNzL zX=hjG=x$oh)u3PHj31`(Z!b>O2f}_`ZF|5$=vuu^Tb=t4B=or_es6ap$Tzp(5$jaZ7VoqS7Foy`Jp)e` z(-;vGAJ%-<{Nu55<9?bh=krmo{4fhve?-u zn^0Mg^u`kOHUx!&G{lSZETGfX&^CeaFgKxeP4?u2!TCihY%s1R?}x2rNZcl|noX{c z?Qvtg*Fg^%b#R;<$SqIlwAP_}1a8%0OIUx~2mTAS#KHA|pFfJY^n?96tMAv1?`z2R z-5Xh*_6xP{zIfQmC))37FxhGG@z?b>9`B`;&^wfcP@XkquSfKCf?xzRcG^h0UJ9?a zjM&wxULgD0LXz=~eejD<5fzaFa9YKrS zDOQd#q+kl|NB>ux1o0X+1?`h)PM$LUs(eE$$DoBRf8K0d#tji)h{E__<`;_Cu6$Ub z(vg^TC7IAj{s~9uC-r1Jj}4R4)kKuTbNo=Yv$_iI$468Wh~TX({5c_0lb>p~jE_G} zbOlNo_(zh5m;M~1WDD#JMDfSM-NU`undXNZ-iy_}gO3FpskDf%&sE;O`Cl3lV zz%+G|w;PJ0Dq=U}qpV>M$n=iucqHMdMUrK13uTUuGMv08C6S#t7OFmYOOu`}_p zn_DvRvU0JpahqGQnwy&?efszjWNE>{!D`N9$;8KH$;ZUW#%sZ3#>-*J#AXg$$;-pb z$_c!q*GDsC4n95}E^ba%j-;O-mqDhc+!m%Btei|7rX0XP*f@a8SlP^&_}EzaI9NH@ zxp;Z4k~YBVaD4n+e2r}MHb`(`5#(7(*=(E;HUVaaK0e^K3@BL%Q4ODs%Whv6QthCV zm%t2NUFj7=_&gL%C=^6jY&a1pkZ3IRx?;V)n!H!%vXioMXZL}vK$p?zh(eUPIZc;+ z6pcge^RJOA@6iL7HMSP(4yIP=mgR<}f~A6?5}yx4qNGECs}UVHG~Zewdss5J_?X`z zQ_2$X*u!E}_ecJ_&u{NLH}B+m1^<5!$p7(C4eC^OAGU;()e@T~Y^^)A(5J|0rpeJV6(_rpLP}3PI(}$Xeq}}bV5-ag(}SPGskO83 zJ|F|Dpsf1YBor-JUtfPQYS7fYT%%#K;d18We!1OXCSly-a@b6@bn#3M4-ZeYbdk?U zSC>yqQxoriSIO}{3aXU5zdLE%;^F@OezeB8_wTNvf`SJNGxNsXi=aF`E#stG(zr&; zCgh4QYA~U!tPE-G>M5hWylu4GQ%7X^CSVY39~ z(#4j91p68b_WAV5Bd?D*F)J>-1{4@k+4lDK#{ojMNg&yelkAhOKg_tZ+^?>#K6Q6@ z|6`oIgOnsQSK>!7;CyR7P*PUrLC3`WHv;j$>3*JQcRzWt;fz%)Z{kbOTfH#l&N5{^ zHRUA6Yu#kx;^I0wKR*v;PO~4dYc}%a&!$-f9?Q$irlMGPnCh(HVl=4f^YaS2tZ6$u zNo)(vqktP(@LENc6=W%gdT?V`I@Qq!>3!bV=RwHdXe%zT-T?!p^z|28Rc?a9NX< z2{Is62E10Hq2y&7POaS9o!8pGnjMKJFI=Kne-T8KuDUBaZ*|6w+%6+f33^_}Kj!|~ z{r6M&k8qA!Ib5ox^YVPFQEWl58U$Xd5ZTZ$yQ#2{=H}5)yP~2(sQmk0I_*7h6JYmCmK7Mk)qrq?lghG6P(9P758%*&MrEFro0rLf|>^^rD zid)=CFW(;hAZzl;OM_J5I2&4$rEZN0CyK2oRsmfSy#psPtW=@v@WCwvE>OrAPUKY- z?wgt%u4KKkvhr$eO^sKSSbmQRc!>-nDyyWlw0dxm=JM_i2P)W9hZPN0Y%DczTcO=x zpeOJx1@iCa-?ji58QDK`ZSB>H4S!Af{dyHJZkr9K9$t(@)k^?|U~PA{Z_jcFk7SVs z!_TdD_e0<*=IH2XFAb3%tpVoAvbwUute~WX!Og{$4a}54k#2FIJ9;>BcrP<2#|Z)i z3ID)P7k^G-$LL|RYYTSU=_9uiv-+}W3-(M62FciAYW57q{aeDs@x#yCEQ72-LMbt{*T73tJBe!56xE1kp+{!Fi_tb~xnNl+ImmAa>5xhXfcGEOcZ z92|73tE)$)rKY}WX=!C@YioZhDJf}MtufrR5x;J644lN6w#7n2NH<|Ueew~MlA~8O z#Lxth6F8J>Ea0KQKSBR_t&S&0f~)K5n$oN0mIm%&+P-*f;o{`%;uI1RT3KAYSEWwi zEnl`HK!+fNwY>{0oN<~GO6fDC5JyEu#>apCN|uw6VQXk;=viA`opEs6#+{+ikgo#d zkbuoyE0M9+UbI1x!k^9psJjl>!)C4$)bmV}NO@>bWK>^jIyM|@%gTP~R$2CJnwm;6J2mx#Jwu_AaWXY-xWLuZ(^H!|!N)TI z67Z)uBqXGyy`4{<5^L{63@B;ZN|h16wJQBB!}o@B4T;Kx@Hq8fZwtOCZx=$ODQ769 z&XiSSdqYFP%EjZ8rfyTy00$`)l4|r+QCf5lF2;NRkP9V1f5vWm- zQBjrn1O&|qGU2kt3sAqXYOi)j%4w8PN|-s}Tr3^&GD23dhSKIA;@u+xD4-Yg*n?cR9 zCx4(IE=o2b1%*J+)1~h7@856~-$rA=v3bct-)%}wL_<@)+M(c5qwK{Jq>iwy9+ z8@?B4Fi;VqC7Q4f8FzA@J3;L21Dk7WZzK!n)3egsj9mZz@l~Y8{up~_Gs0YqOP8ct zzj+1Sxt*i!UjuAAx+oTI)-*m29M#g=+9{9@JC4pr2J=+FW;QnIM5LtL7mxhzLb-^O z4VaECfeJfBKx8u;w2c{c`OGsQN!imSAxsKwFn^rHAtol4Xj~VB9C&r2+ReW2vjXL| z(O2=&FL2JPk%^0&TTM5W{3^_pu#PdztsO1?AcsAq?N$iMw2s;1o3)ait?f7=S|oDm zVtH&hFffA3Wny<;QVyk3j!{?QW>GJ~2FAgw8YxA^>4s^mELh5l>H=WhiaR>^Ph5Cw z1hQw+M-Td^i6C(d%6cHs<>soiBll+A;Ll%`qVhnw4xd!@>fBCpYV~igs8G}Tpyf^e z_3KyUs6p3d3wGM26&1<-n7OU3uFG2=Is;~H)!Nl^^akxjvt%<8f-!e+f{ z+Gb6S1RrXf=8fpt6OsqNZr8^3s?0`#Vj2!KAnnCEL0bLC&OBx+_3t$kd=iB5Vaxm6|2qJH;t z9^LxGmf$zqlyR40-XHxAT2? zem?2!?1Y%HXCN%&1E2meHr%3h6Y=SC{eJQN?HP|SQj_F}Egy`cr>B=pVB}*uaOC!$ zq*SVox+6rLQ(3*=ZV>N6D$PBVuw~P(CYfr7SEQ# zt4>5gz>7GC&E^vo=$K$(Vd+$HnU37nL6hu=RoXiHyfss<29XDC4|8)WGggh3M3LiX zkFN!bXUR9o-5-%calcB{dOv+jd-Ud) zH^bXs$}ln9NvL4R@iD%hzCLW}qU={G8cW^OWI5VQ8Bp}@&Bvr9bx_u{pbOP{CzUKU zuB518_BU?MrNWAe*vx4wU}GgzB^Xi)@>SBOrlf2b9lOMAuZ@_}}AX8Sy4mBd?Cuh2~8(iJpxoLuN-9SmQ`hZde zDjypMNB4VUN_gF`m-D%$rSMlj<8`N2X+XmLmx6%fPVC*eU0AQ#)2)w!*NjsQy- zNmT$|0Ax>Se;zv@-*~|5S?QECMYb8Y^-C1JhN7b4!&<$$oU$@zb#1M*q$E5ZTnH=L z;6wwXb)znEjKtWVKY|E|h~rTdL6EWE??HF0vG;0nrmRT3L4SM#c6$Paf%)a~lEyD6 z=9*3%xeXs4S`3aSeH5ThkdX%A*eaDSqL*t_Fkm62ADswqBw@r{3Gq zp~uG0U(xMziz6hEbIG5!0r#{oOz=d* z7qfbq4>54r+1bO05Q>0ciuY6L%ZUkwmjbJQB}75MRYxSdzu|p>g>dQI9H~xEc)8&6Smhi7Q zKi@Q95w9qj9dK+MpFaoM*x8*&03zP0ch?wjOl18g5EWrQz6cqMEBD>AN5A>S#lp*t z=1&J6{O^E1H^K_%iyLk*Vb4gLvRbUzyn5<1sNt&CdNa`44LV+~Z)jl0G)YzCXNfHs zxp}^DYyZKQHp#bm=Aj6r;?KZd?rv=8oiy7QUeC|lM*>o2ckIuNM=oz3-0gm6dVn(^jdLkac|{#6ebx(nY|-vT}29NsS<0geDZX}n1WNp<3gJxPI%Y_--{ScseZ>0Vr1yc=+Y*C{D@;;e)g zTH5b<8qlM*d5dSJ_3414PI-NObpwn|Lr+i8iUT+1>Eo-aDAs>oK&BS5`q6<~zXh;c zU!nq%^BHgi)-CjMUFAq>HOe*g6hSzJ1>m3`(^gCoV)-StwY48{QMy_A_}1BYcuov; zbXccnW?biO!Yvwg0r!%)zRtHjHpVqOH}~D%$H(X9=U|j_Q&ZEQLBPO9L202Qh^y4D zjsh|OW-p{fNjKGX57HO(n$`P~wGi{+WAwI@HILPWf4g$QvYo2Fe&x#u&yETjLfG#U z$lsI9N^L|68;cDGt*+Ymq^;mtVDf_t5)y zmBOU}EL5jb$0%>9%ghi!=KzVBY} zixjj_uClF-4R~pFRT?<8)`3F1c!l#mAtx`kjq9Y#KxWT5k?h>ADZ{JIQBP`o7kltmcfA~EO@)ns_ix*j5{n?jvWfiCQc^~QWMuCvvq4Xd@gX75-MbW& zu?;;~!>)oA6%3~}hMhw*Gcyu_f$xtS8yoU~dm%ta803mSSRuubjydtQEo3Pxqjt6C zUKxsmid}c$w&Em4^uF47)B@w$ZMHh^CLA3dv7n=)Hyd?&O?Ndl4FRStQK7`;1#)LO zy|7T=9}w^-3&{0v0Yb*S849mBF++ivWr}Hgo4bEo0{;AVT`{Y#J49VO4#(onR(eX& zHcF^5&eiyP$MP?+JGb`YbWO4MW!iZmJ|5#q6mGxd$&#O+e{IzmVwjViO&b;_j`>|# zj8#~87O?yWrOLV2M1+Jk0^ZkM5OX)T#BITC!_vk6i$?|~rs;^Do*olGGD@aRJ|=8l z_1JL|r_stzXliR$u7on+T!d5soU3Tjw#9(CF}8p;wxF(o(Y=8&34QWM17l?!<0ot} zi}(XvQ7lkce;!h(3Uzo@s$8)MAOa!)8%^Qq?M;#U<43^D`Eq6U!2#qTzP;U72n*@E z39AfsLYbGFn_b%GmE9MqLIn^`w{wyn?xfFAiuR%TX#D)dgxs}};g?QbT%+2wz^mrk z@!WtrHi6Hd-Ie7Nf%GCBPpS%>^)jPc2pAdDsxjm2>`WDdjK4@hQL%n_|6MbE(S|cU zCud~$eS7zcF5Dz22omyj^gspI(!A3ryTM-D-RoxOdKl2b%H2jcopedIYfi1B@SgMf#nXIOvt-ZLjW7J*% zDF_YJdBG3I+Q0RYadxh~yB($A0F>9WB^X;wg*lFpux-MrB@HBddbuSoARy2PRA3IE za^`w@cyyh(wKD@lT^f0}F|SXbzIo1KG3q)Uvud3A4@+4!ddVg-CV~XyiVO0<6IM^O zwBafMcw4b+c8xUM$B(16W6ywuS~c=DH8kMd`gNb36t|&2e~Av;dw#o^`N6l?(bBR) zgb}6xAtsz9X?#t-IFUAY7WK$215YyH6Mh`PTc`nscED@XtS1QwNaC+j01}`L6P3Am z>j)Be9)sKCENB#sD z)n9$G3b6%w<>fIetE+171DD9C7IAudI{Bb2 zDw7*XG#Mi!QtdrGIZ8@O35Z=3GKoly>t8;6REh>5P9qZ&6Gdxt^LtTY;Q28zsR3rkJhd-?;9enXgiJ;>M?*ud>IH@r5o$Zrgar)l_j~0kx&4s@lI52brXq zWCtMLk_jtT&dJS_Z%IXk+?`){@&WJI-FqqdSVK6F(_YUjYe)D61eBLoSN|OV>39Y> z6Rc2+z~tjfm9o$Eb#zVv)iRwRQ^b!Gv!PaAgcT#PCPE3-y9*IE4~I4R^%4MJ#to&` z%}3yf;(2y%PXemJe0mx(ot89y2&gRfd&UlGKtm>zO}yb8~Y)0Uo{qs0wvP z{6*Y2$%1(si@0Ix$EWLJD1h>SQzifg3R9e{A{PUaEizzEQY#Pl;14saZ}U`p0%nL$ zkOrmlwVzU_vVx&nskCwD;Fc5^NNsCNi;}m8$B$bdK?+FQ_O)%);H@@*xAQtWUONGr zU)v3=>DA=qW>H?H~eom$K z_j~2#c8w$!9KaenJBVWo<^ZI2`*61C0D<%_%+I^79_hsm4`^s;1fTfgqC9)!3#GOya52jD*zlVfr3O}Wo>OtOhIu2aGokdBO`903n4`QjROFY zM{dY{e)#9d$PR+DgBI+eCok3Sbm6SYreju+JQ8vwC`w!uGe@SXtnBPZy=pCQO-;>{ zfq{Yl);i1h{apLs^VdB9BMO8-6zI%Z(k3&3owTi7=3m_0l;;UmEC5g(2bwx4d@NZl zQf)#$u?Tn#ziz6oZt7!%SbwGh-SoPHmJBEn=r>r)6{~Fk+7WPH6Ds;yxU1-P*G89 zF)=YA&tJZM)1IE4wFg`^Ko46P@gk|P;ApC}Pyr^A53E}OB@%qLQd+AHqi?mAzKO2Z zKCjZe4QGK=A$=obCsEm!9~)ZZOQLQno^EPNm6rX=k((ln4;=@4u3o~P@82q5zyPmk zv1rAeI2H~9HGlEg1yD1f3XL_GvEbe<12?sie3iObIBNou z?vfNW_UheTgR7U97JydPH5e4#+DWMsz`FWz0Bl^n3JVKM{p`=L57gbBg^i6nQG5G~ zsQ3-GCTrvQz zK0fnO#uR{Yy=Ax4EOqq^4QT-xVqs|szYhE#>(>$KyD2=$5#aCx{^AHPZL*t}hv&ID zBO?PSiX_{Rmpe-!#JcE6jc5`a%K`)@4lf?cDCM z;3OX8&t`^9G&m%x>CuIQlhtHZabQmHI;f=QLur%vlx@-W896(2#P1O>VaARhf0QZ^2r-}C{&!vIi+ zF9|Xdoh|sLYUQAeA3w|p@bMLCXlM-mpS^e>j}H$&)|%~CD*&8}6WWPUmnBJ^BA4jF zKP{Ii(X=5zM=~!#E!w!w2s9ZB=E0h{M>W(5vnTZ(9dg%44L~zbLN%Gbcq%(wDb*4X z7m0GTvg{ckpik$tcHH`2?AFdmfC4X-dLz6t$|_++xZrcWwf@i3#3V1!$EVHB3}O}< zH@bSS_TWcGih&7G%kP3Y=VJ2m@&UlA#wnHFSBzT_vu90nU_{Xrf#rao@X-)pxw58X z=WXC-?;o5B{t6yz3-$_Z3i+?$28TXgPT~Z^1xL8G2XJL8^`^;y(13GSE#$f{VNHp( zCd|XrYVG9I6Xfe#z7P0;SIE&39y=G;Fe4LF1_EN#699O;E!c^;$(6TOVc-W@b$~!& z#4kK@BMPP5QU$rS3o_xxY9xZxz_`~PJcSwz^dTa_fHr6M7QhZX(324)|F4C1n(ZW! zYDpbSEnB)6J$eAfML`iVUOGY8zvV@b5VPQbKHu4l9v&Egd4oJd{yle}c|>MpWgXze zz~M@UEZd37%g1d29`5!fpnqg!1gPyJ&;a>OFin?D8c(iUcaQ{FRU>01omx4ytM;c* z_b6~Vasi)WnfjWNPPLKtF5#mS*+#DJ>X0<3jCP)KM~=4v zO4sWQL6(+qIk>or1HUiJN19?G!Hq2}-~iAZV8`C1KVl2E+!Yn&=z%`~4K_>ee=FOy zwEqb`P9B~`0eEVj*gY)OWXf1-3*DXr9H6-pyL&SVyaDiQ?U7mhL0wam0#Jp1D_+>Y z2jV|IK289raS`wtutv4drFzv*auC4f0)0eZv1mX_0UbXZpubA`Rf>N0UZZyP6X~d< z5)R4=kNbE9qhKkuD7~ent{1S{eoEQ=nn`lS20$`a1>-sZooN3?83zxMvOE9z|Gk$W z_AwK;@70RQuv~*ZuD~V@G@d+dJ&6TW)!Rlj-#Oa&?g2pBXnn__nbPt zvQi3!_2vFooa#cgS|3Zl9NWD9kdF;Egue%QQORA#8`80VmJ0QI59ZcmZgX0 zZV0~Y3q2cZNonCa8_GyEE4k>JMp1SeJsxTgjYpdhYswD!uC>kR54n5eaLbvT{7#vgdMn*=&*Vk{xXJ>J~g56HJ2%9x!rGM%1 zPt&h}()lFMWVotN8^gO;Yu?W`F5rNj?(%<~uxvd>O&u4?fsZO$G2gRkx8hOTfcF$D?)nWBOMamSLbiE4&jcW{N?ICB$)b?K8egh|kZ3BU67 z`9DlzHlA*P9mxOt$|G&api3VUh~(L-i?x3jPA6>m@cgV_WCfAlB%1u^9$-G`y5@Du zR3}0&1mo)0(y}tm|M&Ib|CS)MdeOS?;NTz$sN5Z1o85tb zdU|@sQ&KeL7;t;`$azp}GCi!A4}_3v4BCxm(#djZ411d0N-tV*1)dk*7XWdUEp1F0 zz^j4lm$AGwKs8!6EEn}`W}^phCx&W2dg`_V^8IBm?VAIbM$TJ=)mT${f#+cW%yhdE zTx-y1Iy$<>v_Rp!sm?4FE#~|jZt!QieBakGW zF9c=>Ww9+1ikOCc1QG}|88a!tjtVJAAm0*b3ft9gOnph7Kr1DrC&-) zXAU7Xi#pTCw+LXvV?`BtAGC=dG&MFlsxguNDGv?Ne@804>3Z$^IT(0q)*r`Rf1aJ$ zj_Wh8Y9mF`Ki-t}{qD=-_0{^z*m@;kAxuRtpO54A0If@S{x)l>86qoUDk>FE-dJTOB<8wtsWMrzb zW{jTOW}qklzXgDFwXD+UM)dEhm^thw8VIDxr_J1VF#a$cr+7qpN2vcag|Z^RWSqmNh{SXo9Pbmyr(UZykS(Og%iFLXmo#IYe@1Edi4N zwP^;RG*-7lK$|hiOeSAhSLv5-B2`H)4X$S5YML*y3<+Z@?7*Mt1)_Qu}**mzCvNn0^39 z67jkzx}Vm~|IExJ0nqkK$t+f)375K02t*3NQ!rZe^g+Y%4xi6Eq5$cK6*X}AQ$dJ5 zZR}S`Nu2As?Zhdd?f^S*3TDt<0iYVO2r=(T=TiV%)q-$NCtX->UOWLZc|tCY2hdBj zfiR+@saZ6#e|-mz$k4mHyMH66E>VCS(8LKXgb`Dcq>d|!h71kZ>j1gAqO?`@A0r+<}exb|Q*YTi^xd%JFaMTLT; zxjC61nOy)T#Rw}Hgn(eh%xnk6$;&gMNs@^=iOyu2P6S}r?Qu6LJQ#zteF>23 zC@L*QU17|NRUU?!Oq8ZFV@)ZI>NDgN!W0J>w0hN69UL6}+3{DvIm4s}w8m$^$_?c- ztUV2kHTKEl2}@3Ae0-I-YnztY_9vQDzb(CW`4n-$9-KhQY0xwY#-z6M^QN!!)8_0^ zyX;lqQw&5Bt3;9tw6}g+Zzpweb^T|frZ!uuwRHAAQfh@hNAcGm;Nj}{d)9J(*Kxn5 z@E}*?UD(5nN>b~?kdF;`z}Faec_k%pOluwyt^BsR?TB7W#ez}j2e`=$jErk@w{G~s zrzZjRFN+gU?|z_4HLKFyeglK^$#TQFfRgmSYK~i^n4fYnPz9@{5~?#P+%TeU(xq`Um;efW$;xz=w)byxqBW2P3#-EPRIi>%1wXxS@dYaru+A z)kr`nY|JPK1ky-1y!>TyNF0#2aE%BWxVb@v=EOJS2R%|{?$3ZZ`Hv%vff@;5)g|+% z>-+lp_<&?OCrTmZQXpvEZu{q71FqkMr4mgTT#)8O|BkyZKU3^Q5{8tk<}mYfZ4ba0u#KT$*D4%^$7 zWvL_T>IT1z56kOYTBcs`p1bg5eA8e^q>rDDJK~SrSAjg-um6(@UMF3MlZ4dnI=Jsk z^#%#9{2N8RXbt+Js;XExqj_?6#t7J5;2W}xShf8?5*0judMRMV(b*4p`&lT7I=UK z+x$7ry^*Na7fP~jg2Z^bDccG$_&j}}LY z^MF!y1w(vOtIjn!W{t!^pu4x}GMyeA3_=fO^ia-MTasLPGN9lwcFWw?Of*@v*>_&6 zb6Rz&%8k#>%X^itj2tKo842Lj;0|z04I9>G^Y!9Bfr3 zvLO^>O>s>vj7Yj_IL#b=jgKejA2brZvV|L?YHKPPR%sLp5gD`I#puJ-!am5s`~$3> zHrGh%C2vPw;y(DVW@aDSuQSfG>*{bW-H7O+^z<6Ky8SCFM1bsJ;p7a;&!>XvtTGy~ z+ZlWd_D@Uz7FuU5>o#kO*BeBY=K91Ioh<76gd67Pjo)7h+E7Zd<_@9G2}=28gD92f1w+(rVJzcljqMIih`#cBjy0+0=f*- z4%_*Q21QgUK(``z$hg}-(FuzEMHDz$pPruqvk6-<|0Zi9nD6_Ye$7@Kz&Aj=rvC7u ztC|6wgf|4t8;v@Qk^7Iu+9*-EWO_8@Y4j9&4CF~nIf@M5dWnn&3G0~ggM5QbIvp6; z3D6OVG1?-7ihPljKTFn#yOkm~| zxpCtyU9>iR7ux#{52;NzN?q3r;e)&<0$?Gyk{{87Ux6R~EDR`E^arb0z4M;GyFFByX@Kb^k z%-%`2+U;;t#6UTIpkfGIE<`iP9GspeSXo;3>oCqk2JDZ9``H+=_+kE`cUmnY!1(|T z)Xc-fW6qO*JyVcOpU?H|g-@-}Ta__r`Lut^NUwXO;)S=k%{iL3lLloW4KYGI&$rKT zQQEJ0k(eeytQYIO>__i?^7WXOeFb+CLHlV z_ViB?o4~iujPwMSbd!qO2uU+=LKO%DRfO~K=Ili2E^MW^u(uou2bR}cJ=bxK%sAqu zWWMBr&5-RQ9jst2%gd+v4xkkWyxAWY^7ixHNlg3#HkJ>TR#p>W6fsdFuth!Mvs!?C z;qig$tv$Fqib_7;pJzS%S^xK^WFU(4e(v{gw>qtl`}Y+T*eI*F2Q9${ET#WSV91#9 zQQbB#3_N!e=BJk2VSKUuV^YY3t&p)%#3ZXxeFS@|7IY1-_8dP!q{Q_<`i6_JJ?uwG zT%0{xq-UTQ@h6HDKtz=JWkwC_x6m7V!hIH@4=Hfb*kOJ@$r722Y(Iuq?c(?E3per; z`Uv2Q73m=MFi%>tYa3P_b|ZOw)WP{uJJ+(oX%!>4k2zeLWFVOY*?Gfixx&bPZYdfu z5TVQx1#Hx(Lc0>gT)FYx$G02#Ns&GK*HU1tn(4Uz-r>rDjbfte-ubmKzVI!YPsJB9yGIiH-))7g%u)A`Y5t$jm!sb|6ty%)j<0l!HGqY zkX+p>p1=Kkgki!l^zqD}8p7@;@dJG=T~{BG%6RlAffF7ZV_T896z{XY#tU9^5}e{|UfAk;w|A zduu*m*}(kef~okXB^7{~55}TLjK3Qg{O-52>cDqAb|%LqqlDeNC54I_WpHssR`8*M z67wc6nn_T}Sg}|z7J~f-0rJisf_3?Q&11}-!0CK7TqHOuclhfoVQ$PXueu#*q~K#} zLnOsT$<&048NT(suJ_J@k{?`YU4Dbw3Ow-;<}7l`j8yw6GK6Mj689)nj2bbi`^-;Y(OfOjQ5_~~>+adeXm?mMdvgEYKEO3{cMf2E#+WBw) z?bYeFF3cmgoz*z94(A3u?lS*<%oFl-vcVb3z3Dw>w{F4@02N`A+UK1U#H$gJds+(& zaE&%LVzkkcCYLoM@7(rJAk=UcA%t*4vA6hxUl;;Rp!#@jCzp$z^it`cuiuVmKEiKa zEPY+l6Y}!n<7xT6X=N^MWyO|y)K;lAn)YD`_-+L&<`uwv`@s?VR0wVU2AI@h4Te~o zsEsFay(gH7jmOLhBUwM#Nwke;m{iVn_7vHzhW6XCuJxy~b-zXHH4MR)MJ1z9RQU|j z0liizSHO@R?sXNfxH=m?hB3=KS_5)A!%sP6O1}k}KTj&I`=ku7Zer3q(mZ5;T0v?S~xZyrHzb+V>zWpV=ikaMh z8$v|<9z+>!!8r}{eg~!I-aDK;dI*$lh%S9&Jw0jkc+PqLXIQXX$t+_Wxyev zx2W}d+b#1Rcx_)JlV1ZITMn! z!rz@$z2`c;V_*(1OS?~{kGI>Kc792^4`wDFTD!gxgiUtC9m7HV&0RY=Nip1qI^0Jw zhcRT3!-k~qM#)NrJF`iY2#pn6QOLw3@0@RLeb}9Up3=~+O9&?$K`FrVd+2IOy67^8 zVa-<<^H-sn++p6>^+J1??sEC9rtY(yOjw#X^hvgn7~4>3|GZpKHe5Z$95 zwJKV?NJB$_D8wXZ{*8XEY43HX@K2w7)wk0@$quMpe?4BeJLkUf`VJ*9STM~|lAhVL zdsrd;yGgn=@%LmR5@sROhqays7EPdHuD3>C>Yyr9=}``*d5jA%C!VLS-ute<`V0@L z1MFbPhFHrdO1C<^*!>KkSM8Y(dp~f86W>ZFDn)Ay{(z;T$QJ!U(;Xrz<)2bVK`s1A z^lHn?O8{Z=i%B=(z-Q>xlunIFri%0?#)pf!;8$+0=DSs;y1dpU6iS#Z$5E6?zIWb8n`YcZ*2>a^JQ_btu$#GQrbiV4L@p&GiwUBk9wB>*%m`r$)P9@z7E;zyUky> zF|11+q{NlVna-66oN2{Dr4G0NoRN;6UPpCpEd+3As@AgnFi?FsOc*=ZV+q(|ic4%Q zVcpVwF(;d2?IwYzJb~u5`T2(6ZSLc}=S_2z^GuTd=lPfDL;o}-F-;-J2SX0c?%{fM zaGE$!a&5Nfl@s9VE$c-YySofPBDmN~>|$bZRqoNW zIRf$kd~oLX*z4Q?>M#;`sa8#ks$sleS`nES%{HFC!@S%t8=?QF@8UtXxx7Wp8kViN z%0h&TevDAoJZuik$Iz9V{q<6EZZ=_^e&xNsrbP(#-W*x|RZ;Z9fUcBix4T62_pC4v zsUXWXDH$f3rq6*rSdtq(yfbguU@;8Cj^S0Bn9tZx5r$Y%GR?+3s=Mq{1sHf0F>E%? z-_50!+CS2UECaT=`6jM+A0ORqt*u9$;$*xDeDnYTgV7^id9`1^8vgL-94c_t{f3FV zR;vt(K=}GEVmI>3pCO?i4=Q>ZWF(xpAA{#HI5+#e$|h|c*}e#~ylkw}VC(v|y@An-IhH^&1SRYb3x&_YrmcVx2pp%)o1}T_G)W~vS#tTMM0Szi_?ihvswNqt~Xk;eNAxoqLC zeYO=|yZ28tjxT>GTY_>+H`;u~e#n;zVhdo?h(1->sIL&sPsG~~W(TCvm67SEXxuL(l|DD@^y0Gx2&U;Md8rIXv zgrw=Jrs=&BEc1!N=J^_F@4^l1@Rr#vBZ5D!#C^l&?4O9rLy}dJUSpx|z!^(T8xt!ub(e_!Y8Lm3dRmc<<3RJvT?K$$)Ec?K_OpXGs$jx=VxI zd~K_*o`Ae+<2-rDUqgXaL^O^uZb+L6Etp?H@)-%Yf9VMpNlw2ONfglLt<;WiJ5>A} zS*tG66D#2Ynw76-5ytwo_AxN5|V&z=VR zK`FknlAz0zy<^+(tyZLvLd_cv+t}V=t%&q)Kg8oSmnkO)y0miZV<89hv2=qN5$eK4`Yi*+>09Xam ztybQY`NJ5D8mBt6^Tq*Q(|#X~yFoKN@~bXBn)V^Fq}g?H@~shhDuS21vTInZ-;9yB zr^6vn?&XAwxg$~d-B%8`KX&w`QG=&6GAP1Rzwchj1K+KF&7yvlhVA4{L=qjA!dp`q zj^*>5sw$?sqmxQRMwf@YCom?C@V6EWcmSAQg7G#EvVyagF0(4IFd^g!<03jbaKf3yDHm) zXb{N0wSv1AX7{a^3JGT6)TKG*OH17w)mRe-_^r>jfr2^MWgMKU>Pv#0^*r<_eOe3= zQ-L&@X2#N&alTYK#M+JI&~BQ8%7!-dU{^K%8;uCr%ce4IqWdP5S*p?xYTzlv-t4ST zfJOiT%(7cqREEyC%a!Y=!}&A6a%xGirT&@smH<5Y=2s1{L3C5fU_K{4eSY&NZorw zYA`;YRX>bf%=px$VS_V8jO3`;m*GDqL75CJS$=>K$FcOSwlGU|Z0BOv`=2#|W%pnq z6My75D0k&$cm07W*u%<&O;`{Q;$t9IYl9cc>jTW73U~#l6b2d<29os{{j%jDp6+I4 zSRwJbeJ0}hJ+*bkLWfV&`xh$&wWE&Zn1t961X7HThNk3=N}oKvg6QXsN;M zFi1CAWBZZJfD#5j+9X>+k~{ycney-SBvl4df|vn>rIX41)+S6qN6V$j8iAu^_-v#BtC% zT<)Nb_pW|6XN}x*^L!s^k>*OI{x;3ocYzv<3Z?{bO{ZMl%vl0Y^6{>}IjsVfe{0-U zrlM-|Up$e2MtGn++^^wWS~cXZ_KRs^1|wY77$*EhGx5TzBJwHT%1ge%(7*My@8U!D zS^JqC#YgEMpf@)=KJRx>F);=Ga?RP_q$3)zQV(Q~H%LNXol8(Vip9Ez2Sc`QwtgAx+3^5At=jMGXD zUnLr3P8U@xWP((a{QqWxSIFc}UICU1N9td&zRY3C_c`diPi7ygYqS0ETwkx^x@=nW z9Jp*idB7zBdGyS5TH0C5faN^#o(y*8Zy)8uj;VFVZGV$Jem-5c8v)TU_cLb@V^#kv z2|efVwSQ~jk+MoH!)@GlqKWmCiSoX3sI=vMLeytm6FL1jDuOl_f>v~4-4Fap#0@i% zJ=By^ieG>{DPC2IrC?h8UGju8aC{E2xm$OSEl}ZOW$`u7h5PF7AdsT+Gp}vlRCtB! zLj}IFd@LrtE2aHFyL^kBu6m%LSR&FBRHrc`k^s-F%Yjv|Oi`8N4`ZwRkPyv_R`d}$)19RyHO$QNC ztyp}6SLds4$r&yVzctdWQeA@ouI*Nqm3%IlV0?GcEO%O8T@1deR8s8`!cO&2K$}Z_ z-E37}Vd&i+Pd6H@pEm}7R6Cl?7h?#7T@ILX;$-(VwB%X2`6mwlN-@eY#A%tWV!^td zutwdmU-WtLog7%@ebNb@-xh_&pb5=LM5j$YPJ8n5$y1=IpYfI-;+yg2LL{q&9f+RQ5U2(BJUuD2Z!G`^3OCG}JSty;}&SAz!-u8|(n>4LZ z_Mi)Q9W}he?|}?b571V|`kUJ6-}pc4iXx$2f3VaG-eKq*t*Y(s-Dj>0B}H0)&>?*p z6*WpN@G3@6gn5xCj0u9NPjv(eYGq}m;-W6&53OE^^~&B%k1W>AK^ZNqWR=FKYD^;J z!b7L=bl77P>4furVUN(58?uYo=7>c7%nE*X5lg@^Bi82dIn>CS98XyBaQT3c(%2}g z$a{0BTJ-DWK;CPxNkR{aL}W1Jl~y*Yu1_Vnu6GAqlt`~qniYkSq)igcdBt@ymYzD*_cZ(Q~8X3B}%!MnRp|q`dfm()Wk!Z7Iz(s}hN}ZBGZf4N<4XVPZBlKt*sV7dRcLt%FD#1ZBUjsLu z!^8d(NvY=sB03G?>_3gh_)&}${KFN_+8x|`!!R%-J1*YD z!#X*6uU!9IWrO%l zTl|6~goKnNQNaCN0x_teb!rIM*m|1mq6xdH;)qq&q{@_KW2x!A-F!d3n!dTB5igqa zV|Br5-<9UX0;9T!N;G3atg5ssevq{-d8?2Vl^IX&Dy!o!5?{vH!jHEHfI(_^s)QzC zNF>tyvT9IJg}bw{H2$tvLffaMA%8XE%6N5ksMKCt5@GcE>A>0wa&WJ&Vr?!=nK*cH zH{_k^Rg+~bBo;%}rmg1q5=NCE+wQb}7zC5rioTR8OzjQ=Xu#DH^!Rv`+|SEAjL4Dp z$Qv!%^#I2)d3!fy9&gPVKCsPl2A#%|LpwH8^K-yEbfdbFgVdl#jM+UM;hWixI~XyF z3C&g#cGJQHQ(PzToqAq}x0rvk=sc{D_XO{o6&m*2LU@$>#ri9^n_pQ#8bm z+xlc}dOR|bFg-2AKs!%LC5@uW%#j{;J?x}Ok-^+IIpH_ge!BQ2U@?zsdA|p z1$UbO<*FGUlqlbk0FoAehZJ#4IyeM=`aabqc? zr=jvPti9_AT5-9vwJ?Ede^;XH)SSpusXL}AGZYgR*H?iXD}A$^J)#Z8RT+{V5}Uc=z)WN@@YiC&FE)f84Zrkz3XD z4{eM@9DxVIRidmJ9p{q5vDtY;3Px=mrt6{Nyd-9ZxdhI|otTwS^O)WqIV^v1rJ+~M zpFdlHFHtQ@vysEMV`@x$t-tIetd0>zFcpRA*JmD!x7OPE#;kuZk=-~n%IWJc_Ao;q z+s>Q(&yvE45IBA=;*YmTa&|W6sbzd08*%=)lO*|{)qA8{{l>x-glte8xTc+KVhyps zyGE8S-ssl}ON!Eiim7lkFQwUiI>DAPp*Bl$yWT;Q*U8HNi45u7uid_Gl)X)}ZXfjI zHKS2E&*Oy1Sf82q9=m*@Y;kBB-P)o6_UDAc{D?HgAhgKf42Wq&D+{V5xJ?i&WOlik zLgA^86P#jjsEh?weENwrNowq43@amG>yi7Yj=Z&f^$b;2+U?EPcBfbKS#6uQWUviM6+yyeeQ8l_1x6+({mAm|$}5_9j`7HWM$U{* zlQ`O6ex+45a>Ws`t>~>BY2&JDl!>F!O%wIy)bMGkp0#2}#nBxj;^!^Rc5P9W1M=+Z9@$l*nmF;&#r)mnsz=n- z7Kz(y@=jgW#hsqbKD_NZUZoB~(t-uQSorX^JLP>1b+T;>{hzY&?g?h8rXR5a4U6^+ zlk9m;xNYw5&zB*gKJzBh6{>1Rw23*t*1ngM`}p+q&t)Q!+@oxQls(YU-H*$l4VIL@ z9j)!&s@;D7KDZ8jzlv(ybS0SIqg0ekA1m6(FX%XV+lhBkoSI!~$uaARtQiw=xBjQ& zliWcj>hAK{0?cEqZF$MuiUlz`;Vmo%Y+?1$t_QM)EZ*QwSb=2sVdb{t=d%&4_3}YC zKE`(`#i{*(7Qbk$E2T%8^F4;&)>V*p>wbK8D}v& zH37N^oIsXh$O$io!6zXxwXi6X#)^e(-Zz_D+!%fD%=+@qL!kW*C4s7lXXG-41fN{e z^vc+geioXR3q}`Y_mk4VD!ulf)z~sBbQLSne{}K2(7cK6FH=s#ek_}C)m4U@BzWd{ zPGpI^n>=%^Zle6L-eRZ=&9fi?Spc0Thy1HaEIU|9t(%KSvQX*28qVm=ZkH6GjHfP# zQa(7NzJP38tt)Lo*}-hs|7OF2_0qVKUr*~A2nR6JU2 z$b112u(_o^FI%J7#9-^kw-eYPnLY|4NMDge{i zm)!R)ayfNPne{yPZH1Otvwg#}zlCPcMk0od)E8EC6;zC}>H2X0OP?x0p#nv!C^v8q zIQlEOK@%UORH5NXppxO|%pp#!A~pq@P&fYvzgyEt`Y&bt(e#N!IAOC2rLNp34>~S$ zmP!k~hN(?`jZz+~xJ7w|pWL8xep`&a_M{zaJI<9&^wtjj;Lk8KcW!LxQ}KOeSB?k} zL(5`{Rh&vlQb|xb?2jM>Q}a}3+9F7R#Dq{+A&6uj?c#d$$l}W_k7)`7qYc!>ylZ6#=v$p+AE5bUD zaOsR86Yd}=mJVl1`<;hY+Ss!QFwHkP?4nd@pxy*_CjB~$*L?7d5k>rJrq}ig6%8## zvU}HQa!My^pqv*o7gHQNXChv5ujOYw4+#d;MDG6AFSJ_|&3;?;IfhupKz(b*u7w@H z!^^uwL>HG}`xSqzdc`?OQxs~}y7%5W_rfj_{I8e4Ya=0iQI0UI0w!@Jki`FxG=fa| zm2U)bt>V)CLulBQL)BF$2$&BVvvTywq}X}+tcf!ny&C>xEuP@r*CUu+M3*1iZ;1_M z;vkAX=G{^1qL3A?mBPbyk_!0gZF<^c8oMAge%!zA)X2V!-tUWf?e<3B18Zq9gj@>C zS%^t{Nf?_HQ)OglEQv|E{F8hb!wjZf8ZXxwVuRe?#+?gA1o;fhds5Q<>3kWS!)AWr!GnK~z%o@oDDL*eE^MImxu^IS7ofzwgUhSCMqB#YEmE&KIj&|w| z7O)EZS^FSGjf#p&cH>`#)Eb!|@vssniq+8@F($TT*bZ`HexK>QK_BjLN=g6iL z&BEzkn0LDdb+v2~U!dzCvl?&gvVFuTvNBxhvof@-amM*F;2%XE=%URn?7aiU^OEC|hd zo1pV1-=S>X3p-`6ACnkO_cup-+A!9`3i=6p+96~dPrA3dPmNY*32Ka7*kj6?Gpo94 z;87+3t^YYCP(Ro^inV(cX55}fHM*HW+{iD^#uRWStfoDqXq*x~@h3)J8_xVqvzxIQ zo^DhPwFAB5^`0ECXvvgAdvN9@s3VhSy=ajv+ zRBDuZG%2!9CoTx*HUJ9pa;8wWw1lviGf;m~OITdg1ex-lGVKOI(mfu7bf#1>1j1k_ zM36}b{Z*hWbLpNy1AB$G(~ELMbY77 zru8>!99`q}n9mKan-1l_p&&-dWxE31$Z%aN;$=r`_0?^xt?ZICL|(-pE3qH|2R}%L zs?d}MG~AMtp2lF3JTG~BO2x`{OZzfs++4&|QM~?Jb7{$!oN@w2p=8%W<71I`9Q=*ckHmcYv=T=i)bO2~~d`V_ppp0@eA>w(h zASnLh$APoWjp42sLwGZq6xJtyNUqN>2A!fZnZ|S%kEI@Vw+t&60UY7%;?$vGxkRd! z5=B8A!Hnhc#IwEP;{;4UC(el*VLzwGF5D|BeLMcXLzz4X8+`_P$9&&e8S%-v?S^f? zn({5wGU|9ZE4+Ba&Xw;ww8449xgy%AN@EAa0bjQ8I)1gqwEn2euk}b^)i6mM6%I%he*W=;gJ0mSDn;aj1@#^$ zR>n*f$U<~+AM`YofcILMX5(=*R5}(Cd3M&lZ?A^@OI0~VH^knA`ZDwy-73)c5&NIL zeX^*B$&Zy&Cp)OVI!5p_1@*`9uZi}bPN~^+)Kq8_e_PU2cGN_!YJa|%b#emXw*L$> ztD_tkt%kqL8LahV2XmtS6u^QGr}qQ|6j^%6HpDK3&-HcNHS|_jL8@qmMMRGTv)QW! zlUV3j52$<}z_2ah=yTfkk$U;TiF4pK=2I*Jjzxy26qHKtVy@2~QQTA(ehs&1`_jLy z_Vw3U_8yC*%f^#}2>n3A6jAYYy0wk^>%lwDW-Y>!En0GTkY4_;h2}>@gc@8khOf#O zY=ap1{vqbqAs}@g6cnVPp)oZoSuOg_EMV?CJO5)HlP-C(_IUL0jx0wg7zRF${heT7 zVSM8THB|T@ht_2|SY}CZEra}b`w7){`RQrc49k6u*EueKK{~FhfaYX%t}o`aj5}g6 zE_)z!ew*{H4yT?md5T1=zP44g*K!NelY0Ava#1fRTjl)=_z(Nb>n1`Xumu%we`McruW96VtYQ48e3{>a?X@zxqEiX zoE%=J3i8k1*>h~4;|%Bekm8@c#gEVwbGbHgQyGOfdJAh|6FHMh_}T425NQ)H5zY3} z>Nbu+yt}*9kSYxZhub4H{)dfMAntkqT`lLD>QljGh@Gs!$YW%I4KZgQzCE6|x~JC% zlwUo2bsk6KkxC3!WjZ|zce^BS4CmsXZ>ztD%Ojukj&C_&iG}dKb2ccG>ZGXCB&a^U zF2qmP>_D6}k+U=59o{9}rfzZ>Jl~D^`t^qM@$eKWO)6W}(3&SqF46ty!nAoA+EXx} z4hD%;*L4#+yGkrE9g+JVgW5@LV&J_)EWvAM#m(lRJ5-gA`o?a3AakvNLy1fpONT_; z(2!G$LobFZv>=E~f#hgr{@iPLzJ92(CNFcXTvAPt3SM;EV*G2G@h5Qf4d~736b}Xx zs*Ex+RLGvvu`YAg*;^~%E!#qCva}-$->2yAT_hp}yL@9ET744;2a<}ry`k^Gy9(jF zh&(%+cRt2862JF{#Ac^$qobjL^5tTMi^dG=!!XRtPhY;OkWWF_BM&l1PaxCDq|2jD` zPqHqj{c~J33ZEp4QibMvm&uxZ1(UgL?%;sty!9Ggy+XI|msG}U-A-k7Er>nduY;qJ zG#CIK!T5(6H6R!M-Gisw+5){a^hQ1rvewEBhIr4}1+`d6>XEMzB`G_8aYtV*7S~(x!&6Bg#eUUaBURh_DeYOpivk zyUhGFqlno6fw=#UiiAjUR5#q@Dn{3E$XRM}9zD~9`WXc{>kBf7z>IE-R9#buJU|fg z!x*`LTl2q#yz4fsnzk^#+Wn2MDV+H2Kd*d5PEA9r?@+$gQ@PponE-tkmhnSEHjGi? zmv$9usjW7v8XeM9RCLAGO9ay#5E`fNGb6WE_XaP=Cbe;ociIZqb)tqY!; zYZS2JN_j%2J1uUGIGH+LSV4@F+8|;1Q2##Bg@D&(j)G1d5(-|{d%f9bFw?hj+sIwp z5cTf{non$v{{7q?sbtLtv|WT3J?Sx)=n$tHI(&M*NdJK~n*XNN+tTOo@t4JTem$Vs**Kfh<$*MpDg@S>~aV;?GL!&Y9BC9 zT>{VU#P3|V%2V!*c=fjOF>EIV*fnS6V17dpT3`C7GG3A@&P={(BegJi89q%dZkT-k zmS%yi4d*J9aJqQ>)p#OoH=afWpB3@;F=J z-PhX-fAICk{13taweSaLTT8+XHWVjhTP`~!MGzv768OsvGK@6+3_fy7gQkw@%}|z} zqtfA~cQS`?mW&b-5Yexl+Qc?4pNMN4*e8|Gy9yFRs2zH*O(0UuM&ms>gPsl-2UKpJlQ4huJr6<>EZh2?_y^dkdAHgSaY~M2qW>N9$Q5GR^R6izy$S-m)BYU z(mVb=cU17+(p^Avjw#N9A5VSW*g^3NeJx{p`vn&trnBJTK;G$Or6~@CJwV7J0xrd` zz+#0U%ZMUVJ5EYUu2YrWUv+eb=SI zo%LtbcXt7Y<&L=ts;tmJ)?yzy1JZR5L%hu=!P|Rkv%FUKBWjWB!{8+b1IA1dMx2G; z;03b#MT~^3pei9EjqBSL#=i@U*5|X9Su!-Q#|mO|#3dibY>WjoB1G?354aaL8x21H z;5}bI5L{6CGep8h5p)RntJrFZ}n!6QO;Et#?aSt$*&8 z>o9_S|3#L^(Go(rkcS#+mm+>me;lb+Ni<2j%tInKmYbnT8mU{Oe4E`PDfAcPhMXlF z!e2;}OBkW^DpEX8Nj{TzC{BT23goV1H-#2w)_Tj!5ykTsIo^`d#`boN^LRfOwcWUg zkM?1c7vps>$TYfqo#cdpA@Pv{FF8ne(UJe`49jFQhGq^GJ}~o^>r(z`72v1qeLvDN zv@ZkXMN|ceCVm_v@;~}I)dDudZ=Dtw<_LNZD`UoS&d5Rm^}jok^vU~$8}w3WCertr zf0(o!rZ;{Ma!T{ZpsZlaq3LIRJ!fiKnuWDB1RpiGm<~MI04e)}c7hn`d^%dGR~ODs z25TKqZOG*#qdt0`ddq!+G?go0$iOyQTVHR;2LCAn4ff93V)tZ{cV?)&xSd=xc43xa zec7(b_-UgcWSXi5XPi8pxJM!n@S2RjAE5Ty>=11{=TlJt%2A#TZ66!U(UhH zJgi{?-O<6^_l4)&O%lr*?fG$UYK&-UMg;oZ@fBmzcKL*jg~jIc`ak~XyHg`OyJ#Io z&`u#b!!8vIIdS<6vPz&|%gTzrtgI}c#{gskKq5Clm>kX*bc|uBsf7+~qoSk#=c&8^ z3%yl&JKBd@pUGnYViME4kB97JFg7C`QU=P!(*>ZJ5+v>cz=jCJ%lLYk6IxfD@BHKO z25wia`Bqx+n}6`SmnB#5-Q8v}OXd`N1c0%ZS0hX_larm4QM{m-_`J`rnTrU-VxSA_Egs+{8XD6gM2aS^`8Oi;D@A0&b+5g$v*6VPB?kMT-#rAP_^NNw{U!r`p_I}^spleOO-OwG zO1x>83__?&%gYv4R+Lzwpt?$s2D|(G9+Yd_a}y0#drG{mP4sNlt>ES3`?%Wb9%sxN zs#BaB1$xNxL|g<>!M;h(5PN*v&^Z(_{Y$B!DitUMZ~CPl-eQ7B3KZ| z54R)k-_?$M`_qH1FlcX#ZE!>XdYZ(t2e+Rd zf8|WRT@{BQk3Do_ydo}AmA>=%Ut8xHPi6e~aY=UJWMoqZg(yTe#}>+_LRPY~$+!}- z!|^|4WksB_lARG5CA*@Ky$Mk`*8M&I`{8|iCk{R{@3!qax zNQ9>Jvp3~Fz}LOLO?TyKJwYR2W!8sW!{Jhjb~x!Te80B79jO8x!23QdT`%x4_+%83 zQ}dy%rnxrkvV4j0-y*yd?O6I;4;>+3E)1llJrk$yc+Gkt0TLjI$*9)ho00rag8@_g z2K$LANa(utnGGqGVWSQt8%j1dSjqwYOL30&Z64%Id1bpPU7D{S< zO%s6x*AcJzFP%*$?(U+!Ngg;_enAp)uHcBt@w0> zx!c24@Iy+G7pc@551)+c{2jTE8vNkg8A%s_cl>~aMCO|s-ttc|FiC&=1O0KuQ=T4&1ol@1#8 z+C3&a==#56nqW!u>46^W zrma~nNgq|6`lokAI^mKJ_+7W)#1vL2`=)aS3WAe3Wqmis*F#cYWrrZ?P!i50lFTvc&-OA}l7I@k2SG9GCdSiYPyf;w$GMIy2(|;AVUJXm70OGENY+0YezOeiIP4)PJ7!2H2 zFh)Rm92_0(4n0XR(ikXpr`jb6lY0xtBcWSPuezbxxv zaLcy*W05Z)RWc7vS<^opvghE4$2EUGr@xRE|0vhR4%4t);xhsJ`eA*i z5|GxgYNJ)pyZn3?U0Nz4>H1R@jpAjbp7L$bUDM+w+q8RHqh~o&Aa45~h4be(WEkPB zHoq@&Pu{W0-n`Y?P)DB`Rhvnz@Ja`d@X%HjnFWkuniekM+1-GW4Er|L6;D5=Fa{bpEd1*e=#nU))dJ zyeul(UDo)I4LY0o;GH%XV=aTY1QIyMK7YKMnf-zd%Ef*=UbAv;X?ItwK(a3EOU{() z(>kShfTyc?Fhy7Ay@VpNWJc<|1mH;9-Uf|BRZ22aVCyid)xGxD9+Jr0l;~7^-`mXD zyqR=i>Wtca*F683)zv0lh%1j+h{{2AQ|6gFK3FDi@tn56{qj+lFT(s<4m7hd_SAe# zye)-YwVc#?k*8DJkf#$fXvXcq!E-qU#m^q9kYtZW^c0TiXG%QFO###a*lU|*$Aq%a zG@|c$;LC;w7fA>bOC#z{nvybFlrv}BdOdw18guqZ;%IM8!oSS|(N&&X;$F8HZXk!V0*btB@qb=HJK^e&%1D^c zCm$v|@t1@KRQT`QyC0H9CqlcqF5o7XE%NJHeG=OLWy1@1#8DJV8~xsQ+L)`|5omv8 zKFD%$7?svjSPsjv$Ko=2(A;y6wB07n#{ATYnDfSeQ9v6{=q^4R|0lZhR=1s40RT}4 zAuxrAD(;MVl^VX%8xNs$XC>R@Vh`e*R2QJ?7aDfYNuWUJ60-y5S&4QNbV>lgupY0U zva}Snap&e*fsskTP6r2{?wE%M9y3`jbkLh09CWxj9=>u|NE2jj9Xf0o7!Qp&M~ewZ zQ6NBbmX=(xqSH#a7;VTp&pbo>tj7bH|4Al5zVL}@o$K2J(ldVFHXF-UZHK=QVP-ycsr=1sbGY7 z<$Y!woi82dUc`u9&#)7*cr!m$gPf!Cw{d>Hj;xV1)cB?uWZ@Q;&=rl(XYl2jPAmjGu1I~jPB@IPe;;Rc%XLdaNZ;C=# z(=Y^J{L+E{gcWx`(2Z2Pvcnq&YtC!U zcF%Sb#*f|j>)xUV7Av-Jhgm^0u?;`IvR*a+IqtDpWMgsn1ZvXb9NHrzSI0idz+St? z#K~fb-1x}K_}lV2n6yB;sPkE2hU*(D)_`mCv8Cm)oQ%xv;PIdLPA^g%ac`HJemPHx z#G>6aXd79YD0ZLX@zqd?be1~ep-dK$^r6k@n-Tx)J9)^8bL4g32STv}=(oZH9A^gT z|IBzlcc2FNA&aQxY?{N1DdG%83%t0ZReU1G9_QRz_t3)&t)m*^66ER=QuZyd6p_K=nU79Yq80oBnCmOE5_UQ;aXp&#HIUMD@K z%(M$z=7kBYc{jtq(WgXLF3uMJxvYsG=-a@lg&k?|bN@RH_Agc0X!NV9irb9;ac94X z!WdCab02SV2f!tHBDl?nKly6f=j$%i84c+kKV4TdpF4PaM&~Y_+o}-W!^YsL1b2d| zsLWFy$}-lshWKtvRFu3*)6LQiUpeVVt6==QlnE;yJWc#Ig&NL;QO8GpKEgc3lO*9PF9$`(PoB72oNX-2eFQ>trkryD8l_3n#IvZOAYtv zJNwe$G3*j}2=hw{lYNgTo3+AttjB=HbR%KP9(t$md_BR9$BR5T)Nnvs6y^poMaMtLW7<8L z6XM~Kw2+OwfPL9?e7Nqly$B**lp?9Gmv7t(9Au_GHNUVR?X%3-C%?l1a%5cX)7mVq zkX?4j4o-m{o--WRANMVYkA9V2t>y3qpXR(xRFoTwgeL0kM-9tuJRYm}3ZUzg3yR0m z;MC3m>d%iHwSr8(P2__jzh^^BR~nB&N#pGgr&%jz&_i416gZ&&H8C_sB>Z!} z5Wnpr&7+R~)1Eh3k>1-$VQQ#s=E|1i_}{o)`5ckd?#<6aWe=AI`I>eb*Ee@(5ETZ`kbskSd~sjX~RYfe|TQW88<&u{E2$+v88OCZQz zD)LFF?0qX}B@n$=Mrx?lfENZUP%m4dr$x!9`>@u-lUsRWIsk;70{UI1j8BZ!(VtY1Wf%m(wX6EOVUZR|8-ZDcdU1IC zOM_zVn8IB5d&oZbqM&)Ra=m02I$Rl5kiN+4d|+OwXCI3;IosA^l`9&!cj3aRBobn- zjf3N0B%|KL<`|c>&voiZ>Ymk_ha#M;L@FZ)t`IGN%Xg~#fBn*fI0Zx!z#9Xezww?q z?>SyFXn^sT>xGh0m=Es#@p=52 z1u88i{Bt0l03Yo2uQ(u2|I8{{BoRxa1*S9vW3NIA6WFJaVFvw~&u>=Kim;zKi@(TR zvLR$jvOTdJvITrBz-&Dt1lgBVdS&=7W6cuiJF-?L`bX_2RIN5D+q>e0u6lmz3N2$# zYAcH~%Ez8`pq;LL*3>X{L!J8VsoN+UrnsDu3j!Vlg6c5~7ZnKT#@~x)PdIS8MPmBhOFO~h{IeG{SBX1~t%`|* z@o=6UVA+q$*{5+)wmLRR%Zy=buQ;*Y>To2IYNb}|g}NT;Hqi8A_ec1zN8{m5*FNVv zrgN4AF@T)*aY-*6$DtY9LRXYAVuPp4I^QUO zdJ^tp9;U_X`z2z1GB(66{&rh=lBdx#Yx}Is+Q`64r|{n7McmHW#fahs7DjX*C=$%*X_e~HZ4t_?v=?X?Ma^>x=G>mPhhJ;7(Ng0XjWKoAAa7j^ z-fcM~F_AAial&=kt}p58;3b5H?nzgP%0`{q(m(I~6k#;@2g%(2XzNtI^yMIa z?q~=E3gYlCy{JXkKKm7+D$Ukbhu9}c9Yz%IodkG@?u4m-zXTZEkXJR7mBeX-GN$0m z!dySeH=lH3(ZWF;UBg(_Vk2lAOmjMM2BbGqv`3iIIuiS4FX*SuB5(e4v7hFBis#^Y zDw@|ys66Zp&?;r)Ozg}9Ca*<59(^{vCMM6E&?%HrsZ`djW8;Vx#+H{qT2hZdyKxtr zFs4zAPeg^w)sFcZ)&qUnSqr1$e)djMxEcyHOtD2@(ZA7DbD=CE8Ow5o{~YS>@@MIp z-j`GoZPBw0kT*jk)>Gmd--L1vl8bbu&{Uac_D>2#Kh0qY?D^;nMCv3>BlVBU>n_6P z|9zrD;7cKNF5c;{Lj)Qz7jODDuf2(K&!xjX+=6zB=4;ZJIsTbE|wl+ zl=bp8%SDuB&5x`{B~#UxpgSvh%~CR>gp;8xG5i{G2s!EqV4222PEz=|TcZ9s!WQY6 zd@tlU(-8kK%de-iJ>s!?bMky)z0-cB!QHI1;`*<+0arey9)NQ zxYh_`IveTNKn`a@2S?187bUKRRUyC|g?milV59A`3afY9)Z4$tw_z0i8XVhip9uv5 z$`_=s4>-zrNnb3Q!I5_0>}J`I3_<57StZKV-q&oUVp1@a2E|4`fwiGU%BYlt|1Rq! z^z_#^A2ZU@tgWqcZ>RQao$o5uWyq|ql4i0qpT0p+PbEdBQ%q#Kxi{_IUkhxGR;aQ5 z-+dFKna5sD`52JO>KYo(o#h^gEGLBW)B?L2m%l0Nwy^BCf>~S;TSN;hfLJcIx3%EB z!tg3?$Nd9wY;{m)aSfaBWH7uLp$H^Ti~)a9tdWy}Ykc78`|GV% zBa3X8cFfMAMH{yb)`ngU)Fhc-m?rKE``Lk=B$EgVcs+z4vI2EB6=s(y3Be*SFH)vK!Vtgni$zG7iTzQ8B_A`g9uS;!Xqx{A0w`aYrjW13r3o@~0 zMJJn9-Mn0ZzZEx;`_RSfZ1eHoL%EO#?7G+5wRMM{KYu>isd#j?`Cv{|h?&5*9lv_~ zq_q0oJyl&D6iIa5=ONQRY+1VidQ~In+s2* zW>h<&Dc~bC_FIE?MkFDwXX)#<)Uxhn%2^RX55gW>#FWdyR-f0(_ma;y;z*B4f}*|= z-ods^&wXpSt;J&}$A4Kwga8VO_p99}N`I{Ve zpxxyqoVTMPPR4}?9csh0a!>Rl;bsSVrJ5f|OBsQD!qEN701`+fz2_2upvCHsF(@8+ zv^-87Q#e(DXzf}^Vno=j%aSjbC=7!dqNe!=uHuxLYiLro2aV22_fT zY{2EpK3sJmhMypp)MV3pM^)71D7z)y~c=dL`rnhGwe9Ddj(H!1x&$(d6$8?W7E2x*4h2?JTh939ExM4^zDfVk% z2BvX?$e|Z3o;{li{{`I9r5rpSaAY>t*2)mn)jRlRb%+DZi_;{Q7-kN{v<=j{(Jv7`O&%EGNsyoqAI!cL~FPmau~d8 z6V2Hmssd7*9u3HN*ISpL!JVsSX%6nsZhJba++3-fi$*n+Dg97?T7o9C$dOn!hbVm&@6NPA&dq z0Iw1qHmFlbd;T0KqcMAXd!xra2#T;skR?~n6w9OPzSRL|%nD7262-Esm|&GZamsrq;2>qu36FC_;5Mg>|X1$;x zQwTRe+oQ?%R$azpK$FjnBzf`A@_Qy};&Z1WT@s(rEs90iyi(@!c}!vP-C{uU<$$g$ z|7py*D}ymcr!PG)=^-AG+3Q{^q^sl}wedxmbC*=U@9Mk9iU^yP-}~_76b+ytx=6VwN(Nfb>Eydh>@;}vWp3DFM delta 44997 zcmc$F1yfv2v@PxyT!Xv2yGw8el1O!Ew zucofMx~UhLv#XPpt-U3gyN|OanWeX_6$FI$N_VbpvO&9R_=h=;5meAn8*%KXjo^&S z%OL7*$BIqW3xq0t>B{IXJTa?$a>y3JkDJYGc{+{}F$MpJiSi4tOmL$77E7EJTe}fb zGJNN-pNm3x(-`l>7_Z^+N&2y)58qhhxaQpj>Bj$a@5YB7+`B-4;`z+r`}tj+0pXo|S@8xyx)QoA&xMprfH1CrkD)-xd%sX~k!AxM( z;+sCygI$yQt8dg>!`S}euJQX?`0FI=+2PwR>1FQ94A;}#OicI-!7OW4UGKw%=Vor! zuv~a4i*d>``|v~T(ig`<1K!c!(RHs=UyFL`rgbQ*@G8JR->y^YaC{@fPQPJ$;%|Su z`K+Uh!alrfwl5#cxbi}7iRJh8K8nMzUD8tM%sJeHXm+>h{h1lpkT-5*VS=NP{BC~mdA!&MG zdWPBx?L!O<5myssX2J=*KFVF1_V-*$)}CID7v6T*cb<{e0Q_ zWCfPec?wnH1&}NElDeHk^U`c}X}W=JbMxZrQ9Tj(dzRmEbL++n{+gTfet#!CE57-N zE$7skRhCD~-$UJ!83o3=h7(X#UHxtgYT#MJsblA*%K&+rvcOZShKk@@tYW318}GXR zWKHXO;OIU#*Gi zntRw_&P__KB+AfV(SOmR@jofi4*3rzEVn;Kl~X*l?i_%gHP&`MxLuVCU##{56IP9- z9gstpVwTI0n#(uJMFPPqkq(h(fqw2EXLy?duP=Cn8FzN;aV|3V)!UV}Q^K;R)p~Wz zj<{5MmfgvsZq1#_yETFs19`GWOj#TC5BYZB4%ax7#nfE{sZ^YUNo~QN{saT!{HIG} zksn62EKLsr>xYP8B@T-^O57+V%Jq#w85~#@hhAAkE1l{0k(PTu8;qn6euaq;n(HPU zQBW2J9NlfAZ?TfDJN1s0EfKvgyxkvXHD%fPX*-v>jN!uNX{%j#_cGNzE#~e`3cQzq z1FMoOgfG9m9e7T3HJ+El2v<=`{=kv58kyHGQnf=9dSEa+NFc1(FVP>tvTeQdp>UaG zLaFCFx60AhX>UB!l(>iht3AM~3*r#Cs(gp4nndNQMWx$L$2rNR#F%l8ln4>h=y^bF z6)Kj??pzYd47ol!%XFNamv;9v1jT*=PpMstI82Xx@xLSpS-6qp8k?E{@441$mf!6=&0nvB@%tgpt^%fl({8r&DDxIc@Rj#b zUwj(LPN7)eCM-+7wz2`$+p?XB2aq{)|tP^wUtliw|NM3l6+{z{o+;9$c7i zttwL-8;Czjw45NFFx)6hw<#A^)TpWW*;*8-t|Rd3_FWGoGM4o7JJYMm{-rq3*ejY} z=Df7Bd`I!G6O6wXjBmSyK#!d_se_(KDxs{+A}YrPWrbrFE>n!TGAv-IB)6dkmo#(> zjfrI1l3I>Hr(#!TkvG)NgQxS~$hIZK6D(wt*dvrfFa}XU9+l6}{EpWh><}5JSGHwH zehpGopfA5^U?&s_`i%0p;ba7C+|kEn!tp3K+-lwWF=b@?#W&40emkV3J|a{z%BxR( zKgoUonGoe}j|m|OpAoroCU#-j2GyT=k8SC)QvdZWX(}ubY8R8C15cx&2RNg;h|-MKJAg$mqDf zd#=k_#c9M5cYI1qn*o1cMEMTK0J%1PnnX_rj?DS<`E)23Hj=J?&a|DK1##tuHWY*- zsT|E$u%gOJ%RbU1eF2O6B?A+YWR@rB*!jRPl$h@~JpDYBVozeaSBaRE&XuB;|;oJ!R_&k|Uh-%j<--^ssj$b8-C<084!OIWVc#E1V! zfyp~@44G<%5P1>uXkvD#oDsu4g<4)PUK9VGm*a-X`{u8N_E zk=|gYji!0k!#qET7UhRG(Cr`>BGhF;GKeQxo~{UomwsE@zW`Aon6%(6I_7oQw?j zr1jv2nCPdCgafjZ72JuBqQxiN#2*R6YI9>EL&EW!FK9?7L;~K~R z(-J4>8S)aWxM(t}+=C^=!LUgn$Z;<*{CCOO(ZH39ja7g`^sk~81=*}x^G5ysfkc(k zT-^X&9Edmvd={sQFcdOmOT&v|H~^E>4ELRd1#(Mx9ELqVhi`naf}c`DmA(8w?TUf3ln3X zD6U0KO?DdHQnAM&;z4|qSb>>LPc~``^9bQRTG<{)%%2jXU+>~E>dGM!2b3)!^@-A@ z5TdnO4aX$Ya33FEqmahmig9cYAv6gttQWFPZ0}zOuC|ru5tJaD>3E3Gl1v}ReM@jg z79F|NEI9mBPI(?5LCjm&wCv(p>iF@?2}EWZp1PD?;PYPCH<(7cB{UamcaEo0Ur2T} zu{eGWwM;RS6^Q!^Oi574<1$T1P+WWQInV|hiE`FF zon2e-Sn;<4EX1Nzf5Bh)Ioj00jJ>-uDWzFhHt8$aQn$f-YyY6f$N$j`9&l)JZ#i=k z@IXvP2*FuVss2Pr*3l=}k6ocVatjK!0G;e|-&!9bsu`5r*5>9?V(HbYBB)319A`+; zrGupq9of|0yR_5e$64l~>FAT43>pfUvQLbco!834Ml3Zf2!cH@QyX@6mdC^FOx0BM zVdyzf`DQ0{=EM0MljI>EbtAx{&l;_lN=i*0tT9i6pk6{}|CIYhr zBhkv2Io2D+@5RGN+DQ^v5cP!5D=e65LXsBb{Gt{hOzczIb;2kSJCQje_s5Txn7C{Z zVj>=#P@Vpo^jCFT^?rq4w!z8jB)#L>hDSVORO~6PEb{0bj%Is+Hw5!THaCoIZ;;Ne z__CT9J{me7pHs_Q%r=qrX{+prh!@Jt`blG7~k%ccF@1&ZAkBu)7*A#=*WN(oAQ=lZa zL=yBZ83mbdU_+<395bN~j`tbtj+j8m;3d03edqZ)#b($Cm*j~US3dnh{?cTV%E?K- zrJ7!%EpdiNQ=1M4%7t1pL4MA3C7b8Vj9fEkztT^HIZYiWqUoV{3>6v?99OTrj3l0* zXN^eH|HK#`&G|FZASDW_VA|KTN=;a*>ZBkQ0TaUsQth)P^#w`4?R5lr&;zcMfE`g> zJjKt1JUi&G)}1d3`%;Uq3Ykxs$g_I=KE*Kd2!tXXvwahEW6SvFHi??S_ypn6-m_}x z^f;q|VSsb143(j4>T&tY(=0)I^kX#adsXGPGy zWv@mUG5J-#T*ZPkJ3T%HriBWD4A!N@M)iTKuvSv1JMsHxwWojaGItA^BA`#Yy-Aku z&HatXSxI1D}B_=VZwzp7JdQpC(7MaCO3)898~ zM($-zRp(<4XB;4f^rV9&r{bMIKo|O5Daz+=iHRDWQ&xLN?z5XP*q$rRU>+B%ywEeP zFgDU35l^R?LBS|Wf_Vdtw=Q)^2~Nl_SJ#nc*ER}X1l1?fSjHojIw{mX=y6sm^DiC4 z06Xq=PzPsbL&cofyE`|tT1TyS-1iDo*)eZLqxo1J0W>6t;UW$=8;V+)PyGD|Zc~Mf z5#1|LCMNE*aP#hFU_Dc`bPT@Unbr`pjF8ab-=cal;({EA?WkyHMbj|J2++S!Cg?jb zTnj-37-Y)&qhw12!fi;~glhTy)_- zuAO-&OkoWwLA=c*6FJcWf4Nrp1M(935xm>IViw=&8vf? z*S4kp)k2W5PN50v>q@X+`*DWpX9<#2wRrf-pkl=`C`3l&ENxmO8Dg&xu7hpPyh=~X!ZXMi zIEUs-b;0#kCYq&P&HKf;x7na=sS6WvLs9j#_ATuo(c6R6%i-DLn3xs}qk~^&SF&gX z4O(Hg0WDRzQp9n$%xw^nIOR}N1%~|AZvh$l^ z3sKmd^El#5?hD*34StPWSkO8RX-aofN5Huy& zf0%Rng0B;Trq4iI@ZO|oWjn~Mk^&Y+og}T|OsWOt0k^0~h-g!CN2V)k!b)+o%~6$MVftaukm@wUxMEzxg82(&-25S!}@mrih1h`2=Fka`IM#$w}xSM$A2;>*d?su(`zZ9u0v413i#U36WK0I$$m zv-CuPsfw!T>aBuMKgV1YExy#Bs*yZAy7u5d*!P_~L5!+4r;UG$e{gy93#=cF@j)r0 ziW|v59dt}&?1`8lUiep~I2X+Sropd@XEpzf%#BPF`o$zPRkWP?zN%{<Fg9@=A{57BsG;d^B`@4eC*uas8jtZY!&Nz7+#0lkPmU5-W?h$)j!P}n%49qBEIqlH6Qk|czCsfk z)qHP5?HbvXjzk>8Q#%ok`B!Nj9+M)weqvC)-84p}NSQ-h?VgxVhi5^wccS5w21erM zh8JOX$7fShwq{A2r5XOIXGu&#EJp5JBj@@F{2gb=qIxarI~}h)v-?UPN@5}(I5)P8 z0QrQkOY$1yVldkESY@!b@*~WNBBP&08skK|wfKq!;hW>M5(6s2=Tt^A57ik*F_bn> zud>1pCxXb`N@l?=3LfsYA*7_e)e& z5eg2>1SIZ0X5j5q86EVdf4X=u29y0BWTa2f9YQJk`VB42gaXZ+E(tI59+gN}QUD7U zw8DPT_Pxpt;;b8mGl4U&oGS{0TSiWI9*T^X9_b35RG+4FGEgT{WlkE`Pq~0v_IFnj zrtWf*>{mCE%+s9-DDD!}nHU^nX^AX~LbTSFkPKzo_zQPuWUFC=dM5TQ0`Ppy(T1}? zA%a9-NNo+?w{*+q?I3gn9Y zn=eXkCx~3Fs8oIqZ+Z9&m8@f$^cq51nja`%Aiy(Aacp^hp&oKm{7t19^Kyd(oS&m( zS4d9A?Jq%vTg|l6B0m}Aroq^I&L_4j&}N8d>K-Og$`XEBKd}$gPWS>BF+;@O`0P8D zHps75CiYeZu=H1Epw0-hN@=V$;LSdj1q4x>h2)ylYElv-bf@il4sx(19UET|o7lUD zNuZ3~V=Bs^i+-2=WI{hS6|r&2M8B`U5KNvaCn`DeB?>`FA>hl9kpx&S;*(;hhTW-9 zKl#t0KjR!AMiHSu;c?axG8n}fwkBKzvL(N78RM72BnK(TVY_LVN!aL*q*8OVokQCfqqrQaj)JI`*+?XQqW`$zx!@@YKgJk4!;$~~I(!my^r zFj6vyM*r%dU~Yvz^$Li|f%g9_t=q^U83uIvX<6}wWQI)Ru?H0S+eQ8u)Ii-GHIr-Y z8c^Ef(+K-O>5n(0CRrdCSZa#$Zw?fPHzfQTb}0F=0}}fUp51EB3*Qv@3kTB_4t|CI zwO8ik68c#<7?t=tTBWW~Uxy9`o~JDG(mC4+PIm|)U37bf{-1ZBLBw>X-OWaQ1LLV= z+CYQ6&??fu-{Z=E_e3BM7gQBH6DV?)(?tc_tA~PuLc_2hjx{ha=W7$_%yq@y&LKWT z%Pc_+s#)^7UggEZ#AtARTc*-5hM2!!7!~EPw4wy*M$D1Ic!RXGTto2pXfWpa3M& zir#>HmmCkVW@f+9NnqAAl;IxbJGbL+igxOh3b$ebFrT;uWa&R;Bg?!>h?C^$@WSK+ z)dwe!5?%%SqcsfM*-$C9EN}fD@rL+%lG2UbB?WKd~piBnA; z?voMY;ErN9k85dMoUBXT(8%L~iR086Z_^HeJwky#xu!;Tsc6!VC_2l2jHo9H%8JZv zk4Oir=)MYg90!jgKe~LXe`9i8Rd{EIgtLO)dsJUI2|s*B$@98jCRKR~@&)l6C4(Y{g|X1bmEYWlg{C*8fgVX)qa=*>|RF-=(zq?UAukSwSJ zgrqD&4J)eiwC^Gso6@IS2NLI?QMoTyDqdM>NgeUd>-s-FxVQw>8*o1I*K&IDN{pKp35bGEct!`iZp z>Ah7?mzvN+!bH)Qk3I*Z9OcR&7IMODgIsQ^DGY?$v4pbF4bdDHrSd2oWzUqA!M3Xz z?#@cS#SQZd=}DvTm8dcPmTT9s8l&j${yMT=^K%zGg;;ZozuAPi+e>ev(O}DQ_vjzl zJ^VM>tsF2`rv(_sc3{#BFYfxJBn<`REJZt+NXFvaV9Cjaonc^GXXSM|Wr&8Oiw|vk z4B_!G@<@u5`7~nrCSfVP^KyC{z&AEq8uu=?C$Hzr%bxu>^n+g`yPOueGC!KucitaE z3*SdlA(1in#;&s8$|g@Pb&D42ehA3$3R)3euwDsLT2FRGp#L6MQ-7p+|VVfq&4TG93D0dBvo$GV$r zwp1~jT8>2PkQ|je*dy}UX(GpTUynEGv!ac2H^h}o%3 z?4_54L;TLas_t*p^<~=e%F0Jd+M{7e<1*=g$PKBbgZ@GC77-;(r+?Ww*K%JT{QlVh za`r^`QgtXfFEFp{yu$l@a2;F}$mAc`PN`J;#V2oQe&a-XeS{+6BBZ-Ugzc8Zhu+kk zyB{*aK_GY7eo0s0Uw=bWNuEJ~nKSuYfJ`I``+HpfF&Vv&yGJfW9a*J>lu01Q6l_2x zbPL%4#?O)ba_rIs=z0iwq9%yW^m-6FxjQIM8 zprLZEmAt<}+R($nd^82TQ1q}RA!TpwMcz=%@tfvu6+@F5MQ$vw9Q~0@#3%+cnk*GX zj`I6$0p23AQ>8oc5;*WT?098t#-$dpD{;kOiO0_cooY0Y z*O&7ki*7|`WCsHQ21@oC`a`U8cuYcwUxp-B*cYcT*cy#7vD}$E0ZP@0uq%*aQQv-G zolYG`)=hr`#URZ2%tKmH6c6TTvgTWAXW>-|biEN*gPHY)#Z?#F&D=5E6@$iJ*ufYIeG>TZTb&anbA2CL6zmMt!O`XT}#}}X+M4Ak((cOHBe2+ zlx;(z|NeRh9yC8ydbTV3e&+Gh)Eqj=sp6zq@ZbuHDw--1CaE(>k5O)zTL6$)N&}H;u7NiYb9&+gW>A>xbSxOU%3`T3fFA` z`bQ8wSfHX2BNvONSZ2Fdu4%!4SP3T2IfGSdE?eXf7d6QDjdCih(Vytjud~|5!ASmRffiR>_I9y=qzFnY7KwE1*;CKG}#B2rf?`wv-`o%ac z@#*hc^Ewfs?t#c8-0n?qZNfpQ&(iA#rZA51kwfZF7fP^^wHDRi=MxPziM~6oYj>i+ zvIald7!(ohELg+OmyXm?!YdNBiPfFQ_QE%cl71Ahg#B81-o$)mT{|vF`Z(^cZu~*E z3Qfgr$;V~MZpzMVZpqEf%*AfT#>~gY!_Lgf#%9jP&d0}T%3+#PN<~V=$HB+X!^6$a z%x1}B%FM;dWyNe}%Fe-T#${mz{Fs_r@|vd{QPEPdTUuBGZP}Rl*||BGx%ezOf!}y} znE6dj`OVDv&AIt_%u_h2%@8^GczJ=_aPg$%Q!h|)SeUc%np*HOb6ap)GILpR@G+aQ z0sZsvnDO(nb8)hB@$jT%&}hK$39#`tUeMSezzim5E2ikPa)H_4@r~j>fQNh_;mgjz}4Ez%ek70%(#&fXs>h%`W*;b&*bt$ zv*Dw_h4!1U;e-G8`v2uwSP5OMuSTj0 z&4}%X#cVkyMN@5jV3PTDtHwCaETt3y0f8M7v9BEc!O=Z&ekFaQ!tNr$pWRqJQ*GG^;ei!rxGbu(!M;9R? zB69Kb|7G{RJ*s_teB8;*%na@6>FLyBSWtsF55ov=zNV)BBNPM?G05ItQ{!M|X=%Aq zp;5LI6B8qhgoLCynt4^!BvXTz!W#?apU#FgQS8&w2<8*lLlBuW+W>tER6Z$;^I_`L5rPyNF+DEq=?Q} zJ{gJ_O*VSUD#pRdNyX9LK9(zI>hsjpR3OlcKKPMq<`n;G+f-~CJE#85s{EwzkU1W>Fw?94hILv0|vs1 zbm8<4N56*07$XcWe)m@xhAn>h3R$$7g0=PGAB{5Qsj11ynB_+61Oqj-Pn)x|{3V-* zhlepLqHnij^A#G_7!pj?UWPpg)rc&%Yj$>a0fK^p5&q5&4i1C&=Yo$Uf<0GA2>%NB z0-kRXML(Wqz-hU;T}$>n;OeDQup2}ZYPgOJ{Q?BS@ySWHuAUx$V?_mhb5j#nPj$7D z*H$0o7dtyf9YaIOTxD8p>HdfA!@_MY^q`L(ooSa`)|ndwRMf)vcgPspFA5$sG&K6V zySr5q5)$#A?(S#T*Vh6P2o=;}%iri;Kbdk7;w{e4GYPsM{T`ai;qAO!1-BMlUR-?U zkvus%lKS!`4jsY^La8ashVvUDZ}DuoM(Q@QpoYTi+M2{-tzp^ozrh5k#s-cICc}=~ z%IfNBtYX;&BU$=&i0{7Nch<4)5t{XDI29EXaPvh1d|k!G#g_{uBjycS?6-Z4AVyt8 zPtw427jO2j#xj3E_Mh-R|*wW^5e&kQ)Fahyz`B2!Gp$`MGylcBjU^rrE494KOntyk7@&UAm&JO zspQxGdFP8t)oAbj+xB<-c!$0K5;!Xck5$32%lqnle08G+`eT!uwFDqgerbtP`YEza-Zz=fZ`oiKj4h*l*Zo8Rr= zmoyL{HS)^J8}#yj^z#pISg}FhQ!6d>^1m7ze@wo;Jj`r1*=qcr&JzqzPuF%Pj!42d zGxCK4<6?IIGDZIf#{8yE*kCo4y(W0s#VCV49OH^AE*|{&cP8+l?S93sN?l#O=imMP zd(h$2j+Nq<_B^FaMGU&A)eXkvD#Re(qsJ zkVq<`d$($Q$_C<1OVI0rxDy!GTfO;cdkptj>@s5|naMyH;zaY3k$D!UweoSfTCs?; zmzVL33;1gcm3&l9O-=7ctJ7vtg6KP+$!NS&*oORk4D8H}_rto^gif_?k_^JJ#s7}U zAT2L1uhVRJf?lThD!Gj&P9`)Q8BdN;r`A4)&2*rrqoZSXwbd!%>2WuK`|b{PZ{n)* zLZRqRkd&&zl$|tistZ427xScz0F`)9L+AylMvl9{sDg~pB zo=-UuN32OnpWJT_rat85iX+FYXC&z9@q zz4Pl#|EGnkE+qv;%TyL;PPiQY+c))SSY%v2Nw9=O3xxAeT+^Z{d~)CGW-p~2o*F~! zylQd$H;J-2az@6g#mPy=90&hC;4XjAqjwc(l`<0x^7E&0W!avGZ;$8pTy{q+q07zw z=PuV9EzBGou5Vg8hL+|v{YRr@33`oISp$QEyciP8C_GpWBg1_rY-ia#4*FUqCS+hc zlo6k(sHh*DR=?;lVaOAj;r?q<%tKFat7v5fcaw{MXxSeE8$Vkt3m%#H2~d}>QV8Vp zGc%Ws)Qag%348Ihv6IMo$`Qjccsi$d>i>MT9ZcmTJ)0B*xzus-5E{-;!N2oa0M zA2R4%bxfZiFqztgPw+PEvpxG@J zSfL_a{X~U6?nml~y29@C(~!EtAYG|Zs`>&3*uzC%f5o6mr>^BE@VIA_(3;zf@CXoO z83^R4lJ;{#rSoF+s3oZ~cgt7p1#}9w5T%;3^wej1v5Ac%*(RzZN_3^$$yC%taU)lE zcXzv;ot@orGUlp^iZQL@7MQ+`0tn>bEuVY?S1H4pKRihFTW^D5h ztN7v>$-{IJ$p4rq*f2!zcpUu7O1SJet*vTkbXyjCovW8z^WKfy*?hlWQURuUNfx#Y zXFCZD>^q%8EYl=AP?hQdaEk)2pS7zNI;*NCTf4j02SLr(-8)JM!A9V^jMvxKSP>DC z8sF|7N-C<7H6XOg$q;Dy;~C8fm$0n&)}(?0)~>EDTYzxw?C+P_a1kn}TY@;U z02)07FfSKo=9)Ha4cvf$fUQeXG1<826k%auas((yh^hsJ)0307>}6&aO&T zLu0bFwN)IOddHd1q*)(qvJ^rlR<{&k*Q|f!E1*5Xtn_pmV1yMw%t?U(d9}l*j2a}6r);OAgSWY|!hna3op^R; zZwi#9%8m|Sjz8x$3!tl$s=B5c-TEC~{hIr^z-4%JvfuVR)-6pEnVA{c*c|N&&463k znoWuVvL;hD$n(WeCfJDyfn1Q0@gJAKNqB+#$H zbvv9+vi_B6o6kXA;||m_1OI>kUKe5=IRuyd!)z$!T)z5yo!;k#CiBtwgyqJCWB0cA z=d_)PG)5AjOy&a8?;FOzLopb2czRlynwM86@U*6tOVHDQ@S!#Dn=`}+fb9E%7msOf zD^0fQVq!t@hfnWa!Rl2DCv+?<>7q}2SljDfn{)0S9&yVab-MKuz6V)lA6S@}uA<~a zf%!Z-ov6&1EVE-qJR0F{$I*xPFZcw>YHHtZXTy|1(Y6%~vN*rgr_B8?%SY`IBE zNdfpL71i4MKyGZjxf?>33;BD9Mk>q37>$Aj!dU~G`WL8~rm3~n8sLg$X&D)#xY*c_ zpGEw2^DLdGoz@ zW-bPqgHqL%3cw;?eE#*r*8t_`cUyaV1@JiUZy+qMLeWJM^ZhRPTa%D3kM4Yh9=Dv% zTi0A;uyJvJ(1zNxMGTT(thUbF|KglE@;YA;K7&pmL-5{D%gAx;Ja4f>Adh(V=E+Bk zFc3C;77FW}QGAFWLO?!wfC*O-WM_Ap3xMUgY&>z?TD^Ja`%qMhzpO0$v%#%= z*_;`$QPjYe@tb|4YA6UdOez)B9DO1R(W3x4i87d~V$g%(;2! z1%GrC6B1TZOMcf>E}QFdJ5+u?Do(HkIDW(e4*)L3D#CZE0ow!Nxd42k^xtTooSYEP zh<*q?)PcK838tM@#Ka&r0$(=a4`&K`|GnHb(3b+XgqsU6?YdUS_3RGQLAZxdcyt$N zpW2NLqf9n(`FugIcuOE(noUhj!4c3z~A^$C4o};6yNnd@lkuAB2#G_UBD!vP%YDc0cIbfk2CVdDG*kIq3GF8w16%E^Z0jMh3qVPV71wWkGOK9`~fzX!({ zrpXCaW3YjN!Gk2au!Wb~+}w7@dt##M^Ma1?zJ@};Of#z*Hm#P9PMse>LD0i*4+)pLP#G_aGpWXI9Sd+moqa?}r8nVhfsruoNoFis%{vd^TEc zw$piUoDo@VHtB}~XJN!wYMumoG+9oj%NqwiS$Y^4tZE4JH-@R7bUUoJP!<-N4$=^U zpXUnfX;Q`&A`qq3Qu748^oxp%BmB3b;^MSC&z4O|(E2Y*o0}KhM@hW+zE!<7>yrck z2@RqzbrHVD1~BF0@ovdj2Ve}ECkb!mnxKgrC_sqSgKuANW{4Ua8_{{WxP*%7CX^H^ z;3qq+*rr*x#BpQx|5FrRya3t0%h`M#`AkGq^{>mhe;Ux=Dx}_9kbXu+8!nB1{n1O^ zuRAY$4Au+c1v6)+csI}NZEe}-Ol7d8uQos4TmMc@+Ktt&&&|d3JRRg%sbS|02E6U> z@27)rZ(~h;3;Kw=?w@({m_t*7w+XzU@spw!My1ZfG>pEGmJB;+fewC(=;~|XGDd2ar()0AZZ@W8PI!z!EoE~rR@9CA))Wq)I zPtU0V)MxG2MyoGtc6HsgBqyiH=;+{LyQSLZ(vp(4I4M8;IGMxlhmA?_^Rcz$wQu0u z7hVO{?%U*@?d`(P!PXV_Yi+YFpaq6*I<$bdAMI|3W8n9j0u=y+!g_C}1vdaKeK4JY z2-rC^%DLshS`TrUkHi)M$vq<^B-FFVR~!9>0${Kx!4If?t5%J`!(ia5PM6J|01eUS zBg#u4O|EzhHtpyTwlRo^z5u}}ue0=9Sz@9gwk+TG&+~6d^a{Z4ie#Rcn3`&4HV(|2 zwt00Sp#J>%^YYJaWIJu`^i?@SKOLQnyJ^8o%{qgJcD8u}YKAlY7#<#8GYe|&L?kB()eD#e%27K~C1DV*wOy+XF6A08HEGX!=_>Al{MY_|p-KlCSos&Q8uYNo5_86e|IL?9zz~kz z+Ghl^p!9CVE(2f2m(ofC0U%qwD zFE}{(OMZU7H42<$BS2f^yN!&48*_f%x4Y-5SN=%-0d&2Yo{_-|p2|jlZE;wgs9MPW zWE%l!g2=kwcLBg`28FE_)Ef70xPssAc=6#nPZHImKVA$97oQSOf9O>@0gBV8E`}^T_h@ z@C;`vtqW8wfR2uiZf$IB7X^sXis+E0Yib5vh@13lFo}unZ?3M6{{Z|mJuxv+TU4|+ z1u#=Bf!7o9Xd1MQc}0lmXKlZsX7U>d3MSmi+lGb)SZZqOaadT`LhI?=+4R&@B&$Zm zxVX5I=H@%WT~ff^)iaf$hb?WQ#e1nvawARvJYYls9}tj{wPfN5_hTa?e*&PaB`YPB z`kyy;2Y6$0U<7*FF#MUT+*}A(y3z=M!*kBf&4C*|P8a`vJTLXG#l$Fn(~1FnH$d!H z^qb(j_>qStmXuIoVqgTdw6uulNdhXBo<54mwjU0Hl$10O@D6qW4L{g(M8m|Cy5YI% zun464y@|d386#ob0g*)Ha2hlO8Q`0LvA3}SXKHF_q-Yx(lLnm3(|?SN$b3;#OPa9Z z0%&Y<=F|_T^Yp_f5uJN=1@g%T88r)5h{(9OQXl~4fM8}F86EvEsEtWTNx@^FX2+#t zcX$Xc4%S?s6;XjRar(PtDqH=4fVbpm{P?IaCGnMKI>$1O3Ee|J_;$Ogg_7Pze*fw0?TH7>-qWlU-T?2 z%Jd8j*P`D4z8l!u(pOYf(RQb-OZwz>e{SzL5d)n_Spvga98aNLqNbucLqB#4Mi|@pd#WDfrb|H@F>k|{ORLJH6 z;_8B4;|De%nF6iY@LLc#{PwW9S?|0(F!T2IX1v8$+oS_Q zUmC(RDbADQpe!jl_$MSJWFAlny@177#mLAQHEh;RcYl6-9JRMcr0;|cG zLh}VMAJtRq&9TzP#^fekUeVvGO+=JxtMNG^Lte7Xbk0 zF8leD?GUf`G2E@(t!Z#z;0W-h1mIHY?CW*yy*7L@27w~%__x7Q71@sXZ|rhHFbV}b5p!z3=VB9E*@mhbko$*;>X0oif}JXBr87x+>}ya z-lSdvc}ovrLcZtXz}GGR3v|v-&vSt{!1y!3!otFdlTnO#Cgh7R3vry;96<|bu6As|H@xd;P*!-w@>R-Ay2-9sl481nM)=-+mJCA$wG zKRG2DBXa7hcv|Y+h&F1l7&inMfPPX^QoU^Yz87EoDf4G=bBenD+rL5d-dR9j*}dG8 zNb+1sBM{s&x)~aF1IDgHK2Hv+all;~o8P^Cr%drI+U*M}TquC1a4F$HVH5Z3nt|LL z0v0*!!Vyd4FF4Df)e&{M%_Rj8+wixqzO;nR`l#*#yhEc2>N~r;X^qHv^O_hi&cw3oYim-? z`plJ@3~CrKQM-IuyL=Naxlm%qZ@lDT*Fts9#I!d;tjHl`S3;~V#I%6lA_%}(G^9Hl zzg5vXDSIMaDHtEnpLNn2c%t+WutBfgU~?F2jdx$slEY$L=~+W;X(##QHEA`JpSny~ zN;DeY*4EZ$9v&VxhK6}BK!ADx%K9sC3PT(HR%KpIha~dhpZC*r@Y{C+&L76D{!iEK zjn*^MxYAe5;ux+7KWL`3&Xnyg|R4m(AJIGcjpC>)6dxmkM_I^x1V#4i zT6rs9yr2aA0`Q^c%1Y4P#d@c84M2KSJUw|X-34BRx2}Z}j*jU54^3AAR8`lt0cj+Z zlrCurNh#@)mhSHEK1esx-5}lFEg)UeEg(vF`|tOk`DPeqF5KaA&)Iver`Eo_ZigDc z)R@P6^~x|JEUe`0@{(!#z{M8e14yJ`dhUXMW85*bUNW(WR8j(-99zUhBy5IVXhv`A zc!WU7BU{}o=99ovse8zJuiIDid;zR2(ik2sKL0Xj-lct>jFmDuP=uU#vy6}2ynQ}> z`Xm9^fek4400Wn5>*-~KO`mLSYs;r_^$|`pxz>8YPLP&%frXvjdE`I+k;fR63-3(% z{fq!|mESIHb})XTuIz5@BtT~mtV?qKSJm2@2IPPxV5cGdvSf@90VFJ_caIk=b;rQc zHd^V|p!9v%JT$Q}HGTW>E8*OJ1XGb{#=+jff#K(JRa3yn+*DQPNkH|j7X-vTXLYr+ zgDzsDD=!lodXQ99Bqw1vi;7Q5GL@5&vBJQ_WWhj3A9rZWwNRJ z8h)){X^;y;4YQZ6K;nOtII{rw{%WyY2A*A`)@NQmKFktFz5itiXn+3vX_lOn6ig=Q zv$zl1cUfJXQx)JSrW!)=)PVImJlvc>H=|=?8vkL@w0g7g<5In;UC_ zzY|z@vcsBjV^&xOi;I#*HUSzFn=HM6q2eUTng*-@?jj`>k|- z0BemlYnrTDX#bYroZ0;H^1Xw+{KNp}$H!rB0JR_Q@O0=SfJ=iJMaUTefv)laAP&Co zF(lf!q@-O#LqoI*Mao$Ktmh3Gn{AJYWlV>rGQzZZ2?i(5U{X?2uD)ky-%+d3V5@Cv zasw{&?f}?ir=T1Xxg9N!XUKjd^YF^2qXZu0crjtD z`_7gSt}$z@+SffRgz+nWfuK-qdt(_3-de$@*u- zG`MrEIA7MujJ2?I+(-2MHzb!bYmAs^UVo4uVSMNb38g zV2!J&sDuFz=S6x{B6YY_{N>S!y86P`FM8e7(i44sA~Srj0xFl^+^$_GPEQ@YuI4{t z??V$=+9?2mVq;;|^!NAg!1%Xa*lG?L}5KPffd#(~e{;NU1+ z+t{dnPf5AmmYkeS4;nn#xYYXS*$#q_s%nw|unqiwNO$kto$-GpuLjDgCs6TL4G`MZ z7oD7)Ie}sL3b@?e3Wap2ai+VO!jS9c#@f;n{qC<{77|W7`odTaYp$atKyzn|jfhwV z1->>5YlcqxHARreaGlLjnrz`lz4&q{RWG1P;;0;w^z$QihbIm^-&c;VYs$*XYO1QL zq!LDc(MA}Z#O!QDxG5kPaCa&md_aN~^1^#6IdW_FHa9QU2MfEapg3*O3dg2x zjqB=OUe=0FNeKmhiuBX!?a#MXjb1{OK#$#dyqr~lT+)_Ux^6|B0mEMeRu1B&#w4y) z=Q`_DO@h4z|K9D?vy2!up5v4ZS`u`2i{S3I)uKg2kf&09&@+E&UHDK8-iJDl>LY3RIAa`1tr~0OHt)1-w*@->0>J z4Wj({vnb-{q-yS+tyMSye6%E);703()_3|{tFN%IrXc(F59+@jAAJm23GQ|6MU-&b z?ACadb#*Tac6yVNR80WmZvotze#r5Omu&K)(;g8X2B1}wEct~K?o6OvK0KUsSHF?& z72HQh07b5_tV|0*X#Rq!19?jXBE(Rj6QEIY|b%;6Q;6AXgMt6vpy! zfQXR*X?V-ayL@nRasoV2caTotx&s-Phr7EZS9kXlXLt9H!S#H#^`~gqN1v2ysDQs& z1t5!VdQMJGVnTxPVuNM<6j6Hza$|T%7C;P+8Qk{YE1TCD&iOa0QKW^jEbM)g04O6Q zC0#!`9f+fnWcm-jcXelw=^H>WJJ*XQ&DNCn6|ig z-Kb-V%;2)^1BUWj9T6@x8HmU1=K|E;@hl=SdSf$}_c7yMD!_%e`4_Ojq=Cll4r-b@ zEj9J84_OoF3`qB8L4UGh`;1IXl=dBG zA;;CwQ~CVt>_tEZMajICc)|44`d3jGm&Dc#@W2HKW(qVKX9+V!K67?)xeT5;m2z-6ZLZRlkO;A0C19rW z0>JJ1uzvD9$G6FC*xko-Kg(U!(a~`N(g!RPBU+-M)xTH{6d+uf4w<4p0Kt)m9Ptm? zixqO3_%EkK+h%6!a%6?MviA)emuq|xw{@# zMBf(VDinaGVfWRP9&8)0zDSlXk4K>0*_cMWhqvdeK$&1%4Cvp|aJ&3HJ*4M*H6IjB z#El~0$^-*rPK!?;5E%H1lvEx-Sqlq`lGo7d*OY`W5tWLRM}YRg#lpf;077j*x!{|; zprjDn_N>y5b`8>HY)hR4gp9 z{O7||5diyYlkI>0^Y?ES$O&=W{M7!ZhUu5Z!w|u|D{@f=1$hiO1=GNGHN=Y? zyt$ZM2nD1sxSN=kg^IH805~^CI!!idI1unN-Ugjk1+5&o$bcN-E_LCW9sb4n`M`b- zq*o81svmp~n_b(1!~w8FYH8oTEkzJkyrCGz5XMsRMR5}q4Rl>=b6vfd{P)q;+4;iM z@&ubEN?om5Z!Io6yNiX3%dK3mdINp)<;1t*GMEqFM!R#0gS`P6n($J&t*bbDybi#uEzoF*ql@z$L&D=$rHoAaWQWm z-Ucs_-T3?OEEZyNaKBNck&|fz{!uCwRRQYLqmP28hsQ6=#$_#F%UWgX4qBY7W%dMH zc=8a(U z^=s~Fv17)pUQ<4sKU_iZPOKBqT-8sXMh876Kt!^bd(|P(7;tJQ7Z;9dwem*%dm<$q zfTBV&Gl_kGz2pLTDR?yP+{{9WkkYn+~kN$6oPCV@> zHCjqI_K}0Hw;Xr^Kmx09Vqzi{05V5aRn=#ZdYW^0b8`ZoJoDJ_aEo@q^r1OJznFIk zA3M-fs+u*Jo_Vt-K;MY)%>GD)91w@K{d>{~PKKU#)JY3q)dJ9*vaf;T+@6x096$OF z3R%ync^7OP$`$zvD6z|^$faR?UR2cUtQUNWDmoMk8Wci&sy?ueiw+TE%I(qtE?hT(nfc+Pb zt{IOtLHb~;tb8y0Ki#RE7E;K5Oz$#;K-W= zd@;T_>Ng-img3|M8WM8AX%=^G_P906oi!PVLIgP8tlgxlK@}}_)*c?W#z6Ia3=49!#WmW6E&pkXb@nvLy>z)*D<$HtW zY-0DnCwp-aN$3jP_vA_&7JE3CjE4$6|EoF0&~1PJy83z$)zd^@L_Jzqmp3svIs9+N zU*23|{58>_Y6m)Q_}&;j-k#CX(e;1^ps1`2S1^qXx_EtQDh`MU6ioN5t&!B$)}{)0 zbG?869`F?Z6g@AL!NIp{>+tzMMd%NuOsoX zW_4BpShypT5p{ij-sgSNsoh}E8}K0ri)&U83Sdn!MqWA33ngu%2jm6lsKF-G?hLIu zai%E5Lxz1pXYp#~mAar-XQrh|gCH3~ryf~}TKO7K$Re=%1_uYxy^M82`(cY`QG~I+ zW@pEM3!PtCi4JVC#STw)Ep2Vbhofeyw{H=orKP8=8b$Y_S8TXef&Qq`e0%{h{#gQu zF1N03MiNJy?EkZ;`fg${9;FQpmw$jOpVy3YkaVLg5jwVl2QNG_HfGay()ADl?k@*0 zR;2^TTV9ETbeQBz6bRc3e*`%!K<2WAMZW_B{nnP>hv&y1Kqr6Sz;YiGc z7r6}5>6$m@z-WT#m-u1qVZ#nl;&kG;2?+>b+Sg+K)Ahr(CnH4Y+1P5003@8ayVysP z%i^7vE0`V|6i1!CI+SJiv40%_hlK&2qMoUHd3aOmKs^2%_?^F=bGukstV3-Q8tm(>FRw1k%4WBFLZo!DZG&zIKK=?!DoP zqhn-z0TUD&s?U}`?nrFc0i7X&NB^o}RT$vm;mOI#VL;09c96yI3*d70?Cc;o4`j(& zP@f(_pB2;6BAm0j^6%C@Sd!{9sV!Ku_%dh}I(raAEmN~79yasu{Wu2$EN?W79#*b0 z%G4hY2lv^&q0gAnGCi#sOBi^a$A=PloN^?$e;ja$&zFW_parUlDhE28x11yzbuJ&A z>go-F%7NTYAmG*Tnuy4o*gsCXZ@5qSIYg?)+Y_Wwl{7TM)b%{CB3iZp-Pr;Z2IyCc zANA7cZMcX)D+&$?K?E5_bg(#|o|!e3HXXuBY%C<@bkSlGIkrtq(U#k7Yot-df<@FO zKE9EGkYOcZzECM%5WB!3ePj`kki0b#k%f7D>cDz+hz5S30#4;457J4-O;qLHNvOs& z_7;A3ICIzEhR7K^WS<$p(tV3u-F-bd>NjgHL+^Lf9%}O+0$ua!d?ukm14}~H111*M z180Bf+YvBj8<;g}dj5MK-eA}1{Um|d85ywwDt`|kK5NB>@nO@piuenRoQl4m1@jgM zP3R{pFDM9b55wQ=h_Ep*AR8M6Z2jHEi1spI$&RnsG?9KMxSSrF6^y?Bv^M9!vdUr4 zq36Bp=~L_XWZDVq;`2y%53&yr8eU%alPDBdvV!)Whgf{S8{(M z>tO{rvfiNFLftm@9ut|Zw>vqc!Uj^*CgQLk2e2#hsSEO(5ZW>Dj7PUn=NhzooT+w1 z=@Rn`d&A-psc^KA8+(ku|0XYtjuiT{C>4ZwCsfepuz=-(L2pdOM#Bsr8I!a%&VrHG zumrCja~Y>a1&67=Q?>C3;raRbdE8{V=h3BXh=>4nK>z8F`Zeeg-KBIgFA_o`cLd%u z%i{UpOeMUXL84s%`tAydYG3Fj$t2;Ck;U%qnSo4>o*MU`ot?n;lRI>^wKl>Bg~sh1 zB8(j|CEVrqu}vPK;#r|e!bc635mZzetJC}9B@e?VLt+N&5NL!MgF%*!x`2&BkxtyS z9OXl>>tana!Qk6@MI01TMDE|NggU=x;Ik!NQa_p%eecaucgQcptvm0>lJ|^!qn7aE z6W(O+tz&_bvQ88T*NpxUB=y+zlTGIc|F?GHr(O$`O?#M{#E*ESt-nqYzM>h7dGbC_ z3wU4i|2L1I4#7tKefkG-eYb!0|3$D6(A0{;q!II6s=4HXUliI}o@=v${L z&-7HZbCKjb2jq)6Ln9H^Z#3e@vO07={8vxsee^{omWMs*<485TQu7EqJt=Ts5YhK! zqx$CAX~>x=Vfdk~2Nj%%fOtaw8)2>pqIR8U`JiGvh|z4Ie=^HpA*K@O7*s%5-gMw$ zUTSi1cJ2p02I6e7!Vs95P_S975$;=hS=J3}&?uQ|q5s|i<{4qL>7qgP3`FZs(!d)U zg?Zf#)pT}}==+=@2tJ;Q{@Oe4q^liU7eE&k3CLR@%AL0Yg>q}_Ae_zkyOD|yT)r_j zw5d3^=-luciIHzJ@c@Ber6*TuYdXW+!0tZv!mEHQG|aY@_Us%#bjlQ)h3~(8y7xBN zcxlxnDjNM@Awxnk1%b?NJ(hFn?blyq7>dNOG(jE)y@?tg7GKIDloN4sL=3m20n_L; zYU(j-&TIO1oxk_*vrJsJAi}>J=wB%G^()rnMb5tg;C^-aI>I@8^yf|;8V6#DA*?_8 zZ*K~z3(&Be?~}FdJW+*|^=a8OZFrn2uy%`z9Ugzb$`tek0mnNmaWEMIt4c?$8In&2 z$UFZere#Pki1_*+)bEKDvkSo@#o+gxG0-LU&||A`x@`@osm;0ZA(~)8&67l-D=7+m z)nW6T?weN%&Zk#r%cJXybpF=L1vN&!Vgr&0qAmXW??zDFNO*$xKaPlI3@+_|2)wi^ zj7Al*DZy9Cgv(ZxipM6#;UQ7Wr00v~Jn0T?YrChYk6f>uok(6#?RvL9c)<@Z?4OcC z`%ONJ!Q9L&J}3yjs;=%e1gvpFP&Tgu+1~WX!E_lo0inof2i5@Je@-xF<-e@0w?0Tb zCBf`ny2gj#QRzLHU`8-#v$1{0G=x1q;CRHFzKJ)X7X9BV?&cmF;Oy8?)e&{jp)@!pT_C z*#X<*w+eBe&sAz1F$(v31h<|y0-xLAtx}(_+sz%kiJ7V4d7%nO@p*W93K!3Q+4lr_ zDDd#VpFrN;86+4(`X?QkWOeSH@aCp172^eWKjHCL7VsEeoS2-G!kZ`@aG0HH{yCu-1ynv*+r|A|tGXUn z^C{D7oGl||U|eOp_alEmWBcUh6au_4`4|yC6fxx2Nr1>Lm(y4Ct{O`LYQbN_IEdiD zKqc>6K`*bCA6$(Rv*6sXcAj50?2EK^GqM&w>On$@F*x`oB^!0oH_9RqO0v?7$vjq| zW;{R6Z9G!UD`Lwk%zV?820gqbNjEBLY@+@hpf)cF%TtAD!a>^Hgo7no15LqDPx{mLwd32B&n zNEFsvkp;8w1x4ZX4|xJ0pE^vnvmaSG5G7385oMjiTz*~IE0j3cQ;y&FmjPG2=ralV z6x#22G}i>bN}>eA6%$3B&XqW>HKJ`ctSBluBF4}CgJt95!n;}f2ebE1DcP+~`zaJzDJcT%H*fBMA;LWYH>DoF3Imf3)79tCl+-KcZE?R-4Z-Sc z3_0!tm-r0O@}CI;68A3{th7(3%JuWQtWP7wmMbYS@VSk2ba0ZNeO`fyHbqTU*w4+S z3Wi@z-vEW}cGtFfE2ViO96O1|$F0U`*z2n>)n=yo?#F%5e#Zw6)Wv?MX>2lB~z z-0sTpsLisO;B$QFLqnT27N*9IA^8ww?`9*Zx4wQ7BC7Aj}j~=9EgFwPh?~S2Xd)4Bqo@{Q%t6yBLKR!&HM3H2Kfh;7Is}o|bo8{j9}iD3 zJ2mwdPRAsXC-b$QhdrO*3S>4+q}K18O0X4-!uE`O>KF+8Z*}8r{HaB|-jN?_ICE~j z%$Su9dYpndu^V^nEtb0Ye!Myw;Sc&xgj{FES#_W2SwDQo>~1&Zk`PC{zVikMY*b+A ztU2~tnAD{5h|Zp51{;ozVRa=~z=1VgoS5FSZ<^4!`_RzkE7G&yjgeUJHT|jz5Q+hJ z=LCAr%@uS-6Hy;IeV&%cI!%fK4}&x%`CST3U05Swl4imfEMxL94kjOBdmKJx8s6;P zN4bC9^33h>;#_}xdOZ+=)WWYfI~C)d{SVGnf~Y=M&wf!f^5fHntO?D~)tiZU6NC&m zlKzi^OCK$^1gqs<_L068(f&YR}dCJd*G3SnK<5~X8T0owRl{=u}cHP{LU35oc|jZ zLUWk-hSV7rALY({+|8u^pc__8k$rntVUjY06KwPp<8#PjZb?#RK#X}BRepw>6gw!f3T$E z<#+e?zF#>;P{M>R&6;OlJG@5_YAJs)Jv|)_49oDq$T!aIpZk!s&ykW497+>#$^1vK zjqYDYc62eHL`)2l%Dj49SfD+<-v3a;r|Z2+*of+0mH7oh+At-A6>GEdh&Il*g~gmu za6#8ymq2HMm;#2ThIUfSyCrQOVz`8bSv|&nOTCf3{Ed)zSNjFW`bJIjq+)olT;zk=cyB7!hs(VAUzk8)JM!p`neiX2P;NC6X}2xu>CR zYQ#)*?w(`7!lwn={lZYr_v>AFwp0sDge{oY_op^OJR&WEYW z;#}XIF_WPe-0*|o*W8-ijB5Cs1<5uJ84sPSM{j={JRaXyyjmIA7ip^5F|gDRdZB+~jujUuKjEe~Cs1cKY^V*9{GK{j&R^(9;RDduTwv5d!ha1orYHTc#GI6D1@D}n-|Eh!Q?L{ z{u^Jwg-ycKv9^uil*J}$e?&IP{dA}^F{8ZXZTwViJ$nq7n#J^VJ!FJ!`gpXqR#M{J zc~{?aq2cOEqfF_;eBaoVmQ`Zyr2D#eZ7oYoyp?#=OQw zC`nBi5Vn&{RR=>N1*w)S;Xbc%VN)%M_BmtP;LBH6x?GPt-F#EirjDMgv$~;3>f=Fr zzK-n9DE>2q3$3@ky}e;EF~{ZlZx%nB);Bp`#!Z2@~*tPEBDOEn_Wl%Aq#^5gKIxbz;r2S;&*jAdMIm(K1bCGj2l%};nf$ZLi*R7Y0 z5dCL6SbBLpMsNd>l8Wl#^2Oq;DYypIm#*ZLlzs5{XNUJ@RN(iFRTNd@G7hcM)pR3^ zt|d79&h??_1#$zCWXE3y_m-=m*E=GPqZN@Uaz;&`eD{)(;NfeKfFC3l{^vqSC2@4= z1V}!^>Wg)*!Ep59am@W>wplbbAN2Z}OjAE7{tQ6hAa;F2v&9o&dZGO4$lI}R05252 zC|2BLA0>8i>GOa8ll`|>zQhSGyji29%1R4vgSe>jFE2zia{pe|85cuk6f&rP-}S#l zIL)apdS5dugNJzP1r zr0gmJJMzOw+?1tXYQ{T8)Fu)JV*%43)@8AdPCv2=w*^h!`q?1^osxyOuHy_zWi0hz z98%0%<9K6tT?Tm7kotiO?|RZZt_D3#gnw^?jeJB}MyI4cwWCN*)_5&ZjZEYt9*(m( zddgPMZeCl@NboyLZ?*{j^^MFQO*(uJ)5MMl?qw^K;%E)LL)^)WkEu!Q+9bc*`kayi z`5VIi7>h2%M_(H z6bcv}L?>6?g~8D%Ga@GS)Ucw#+ururwf-ACm4G?{42jG#H?*$R&oG07LCtEFeeWVa zv&GlHw6h}R-`8RW;eh+sNAlOyVC8c1KSSR8B&3qA$Nwg6Hj*0tbS^n1cn`K1&x)R) zP*4O15w$>$$d=sQLU2J&6!>uP$g42i6p5)uvPdhN>)t80(oGSjxaHPGM||BM-5hO> z-e}gnwN@9**?;Aiy2Qn9-gS|E)A3d#nknkm!%@z-cyxB(mwb|jfz?J45z+Tn(ZQu< zNmZHZB;K!%Mo+gcLnb<%YzOTkmHqJ$cRI594I3de;JD4n*~hm1B4&hQVn%+7)JC3jH!xAcA8o0Xnp52J&9?oVa zOE@CX*w{zNFDDe*5ux1hAkq*Fjtf9-e((`|{L@q006s2cMtNWw`7N$%i_c8XT8>Xg zZDKzpIgk}c6HPjBn^})7ZPaMRZ8~*oNfxKB5aFh8FhyFZmWObBb&r@B%gr((A(P)C zGy!uTYe9rOhE#yz{lur;q=bBuS1$|ivgMTsGT+%TD=$pRd=9P1B6))d@6+^5UU|d} zZFPMzk2;(+(xVB?7DK-Yn?3k797mPEB6eMvkiSNI*}_+;R#a~>U^DfKj3xUznY1)T z^@%c!BxxMSdXD;y07ph*OP6yT^@^-UCEiw7ax$KeV9xKZ`201=%)eK5-o%r51VZyq zMp3*o9Bp%RB-RqNob3uIB@D3S6E@TzpzmV&<6+|_JvPY8%Fq50EBLulKa19^xEx0YeAYEQa~1r|L)K({GI^$~K#(qfeAoZ7QhNFrJ5%7wBMk2-Kk1f$ z{W{eF?Z?j5XU~d#bW9c8$FhhrI_jZR>1cLi_BT`Xo0qqx*3OP&TM|mDm%O`~P^7Gq zD1?0xpUOcU{g|nOT9*(O8`*2aX2_Yr`sRv2MD+r$EMjthXyF)&R^fcnoaY{Rp&uVHk%RZYfX0@;Bm)FkSf)G>aSM%}Og)8>HIR9{(*?v{( zzw&oiksF@^7a_!)hP8k-;jzNxJv8%rSngAg4OjlcKfbFwYgngtfkbs~Wxrnpqwa~d zcZyH`hmA{Va~fzHT@KXjG#FIuoL@9nO4r_a;Y4|nsMWu!ak*O#Sml5#Tc%~M?ESFq ziWxi9Uu)b~IP?TRyZBbQ!5a!e<)E|>`g0bBzg6*Kc!o4Sjl5y(QSc*fv=KC|2Gc&+ zQ=zn^!&>U!N3m2$Jz(^nkOK|X)#=CAwB+H2Gtrjgc=|-L%Kd^}KFx+aoP8K3*(4m} z?U>Oa=Yuge>cJrkiMLtw;ci4IT$?1E{z`!Z2j^fm1DpUL+F%`;s7Al+OTT<2 zWVcVkoU*uP%Sxb#V=AN(iU4Vbi@dV5cAwt9G!hgJ8b=iPjH=vXBtd~Kxtow|a`H{e z+L-GNfgoL=z7I=QIdjP#gkNQ8q;^V{YzV<%!@Ul?QKCI$1&TAtt{f2P~ zfl_)xSC342aE1@VIh^!#a{gLzzAlO^j^?XAY_s=9Q}i$LpQUV~TH>LtS{kJLskIaX zPN&_(UL8Se4k|=*D?D^ws@`dO96Xb=VTJ15N`;>n+xYF_*A25|`n<_2)mdN0jd=N9 zXsUkr`RDLzQjV2qSQfYVhq?-~_LM4Yqq0_!#Z8d~nl-hGx_aaq6&|};AclQn=rg^= zLV|@OPdFRV?eVHeRfhpY+?qCGUNop0H9v4uofAB7v|TE$o$r+QKDQ6jV~YVV6@JH| zYpH)jHcu}T%XHa=Y(tosCX^ZS&le0P=8q2wF@hE68$?KGVmP-D8F@J{q#4fk$5G6i zw>{t1viR+i=ieLdgp{NTP0_nVTD7P9%k0OZ;@1=f!jt1ES&*c6XhsU&Nesi5+Lw%P zxTjg=#W~6~FUn-DYVl%zaww{(NzT{eX=(F0cz1Y+bomC#_ZrL@b7h1@HpV9tjNWc7 z$r2{9xL98e2D;vt*lZa!PVQryion^3O2kM_9~(!A5v{DOh*GH0VlD=0FrINyHDb8W zX)mAEZifYdtyk|wom^J7eSu_QVOgACmR-;oov6mGw6ZAQ&PY&Z#z{0H6JxqM^)du= zf|BnYY+y+Fpv`-SzpAr~x>}=c@xLuyx#;La`0P@{8_`mv%KJyYt%crtTnSz$a!+la zzxw@!@i%RXcNzzS{_Vi&{Zpr)M98lqlp&B1G8l<}n-4WtCd5;Uw^x54a7xHDQ3hj0 zQ`6JJvSTA(-tuO}WHtoBDy2}V^$X1C_S(6((N|U&FFd0r+cEA4sj{yN<$iL%%wW7R zo}(Wl$r?XW&l(!0Sn5NN^NtE8)BPB2s{jQs%B0Kt+|-?=KZ9eod#-(V`LBL5VEk+# zja|YhVSu<2zVOt;;#xR*o!!Evhs&L@nbq$8z1g|GXS+0#NYpF-n)8lXX?1z|2T0fQ z+1tIimX?moUt4H^neUNSbx&s2jhH=niES(XWvg|BX$5<2Hz3-p(@g21>h%6mg}BoE zDMlI3p4oIE&g@cP4#i%`^x&APf{FJ>wLNc+H8Ye+r?W}S&Y$v$_Se~Ta$VTR98qQC zGG_i5X7kt@9UIf%;1d3R16U>aj$MPZBvVV3&HlmkIvDC_;roAIBe%T;RDVC*Y z=fp;+_;br~zO-Q#U7+4jbc*b+3)@`}st%tX)!9%>JXNjs%G7_J+;od>kze#zIik@f z7^wy2JH9D?&G$g6P+M`6V z3wj4X$|9Qy?{+X*OwLPuWND}nJ27xiA6|HL3R)3qEj5%ULd;9!(|EL#UEOJM2$~QM zUQn|DBXQ!$Vuwjn(!AthE60JW>>GBCmg>Bv-iwo}p(C_hxPOkjVpZAPTZz(Kl-(_5 z`I;{Ziz*A$-yQj*_9L02>Z+5*h63|O>aqz}S^l0>Go6!T1=|)xx$(Pr-i@pqrw9$` z-hQ5YMuu0xNS;4!@AesH(bM{|*AJnuF(enkee85Yt*1fnH{&F;C#VQwGA|z-$UVfd z>w5IjYdtYDiGUW#6F>cbTirmN{<@-TV<$W5^qJE zC?g{yI4#yvx^`eiR>`-g)~w*0cj4N>UKR`2+0#G298>RD(paKnJca>{@nRer@h?${ z9)5RgXu?Xj9@ZZu{|1pL66k#ur}maaG#v>26POvbuy5X9)juS9hJ&!K(~)H?XS-ai zpsLxq3sdda%6<4}AT^qL$+i!=AuhE(m^V)6!GkzbV?CMiR) zsNw(fo-_i*su$RKS>Bdmwp7j2`;eD1_%se>r20lLUo)e5p+=kH@^! z_s`q4wZ_g{Sc}-}OYDc-vcqbd!^Rz@FQvgteKzu{s$dH4b##`h|6w|X_(O#fT97H~ zf7vdN&Mi+kudKepJWFupj+j~U%NokcoGL~fB^n#>Px$+08olMy!mG*{{<fAa#N4Z3m+-{pJJT zaPTmBC581vVQS^ipa0G6gUQ7s6FCNxZ?ySkSe4_tk6&QypjgbvI}>Y}Y5GOuiLone zMOD&&3tBF`!2p#CS7SD#!@(w@^L@IT)wYh!=|3uR_3(YJS;&a;l0cU+JewGfqM4QX zcHm%5lzl`>mSbD}tNtjdgqEkQjZpsX;EVu$Yrn2?P0m>RWS(S1uEFxVIf&jlZVGAC zBd8D0BujmQ9x5nx#OUAQ-q4PZNPQ>U6_4&}a^TYrP)j%)d*>OOgQTZDSa0tmZD~qE z!i;Gb8;g+=8u15F&llqC_OfIfr+p0JN{OW+^gF6R7TnO9$LQ=5YQ{rqzex8li%+G~ z3#zcS7+i})82k28dj7CW58y-woW=x$sz5QVW%DHHhyv=BzESk`?PSdoU>}sP7z|^S zX|FH$pMA&tDvz7cdza4#Hi=t}bn_|+i6ALUN+KY7c@HBZCIa81^9zjYP&z?6k=-t+ zsTBSJf^d_jLZYnEd5S}NX>Uru7RA{v&HU+E_{7-A!0LKPKF&)6CeZM7+b84XFbj*; zFcTs2-`34$LFC{pZ6zU)QV3PPL6SJ1$l$sbmLzQPCGbOT|8T z+*cL^#S|&jzozx>donIxIZcfTYO-klLo5oavbW;4d z{o1x^3LLpW;n=lhzP!BBT0rsa%4hz3)8k9-tMG7QA3PcaV@Kj=OK;&`b_z`A+kI@h#fmn7@GV# zx2Sg)e!dJVRWHk752(d!gpRqF4zQ*q?aixC&os7FOq;&cfrj~^evR|Ah zVX5HQh&Ijj2K~v*lAE`4jysZ(?pxu2qzPl)dAiwSfW#DDeO#?)=Z?r|RI>qFqj>L2oCiHNG!XmB>XG6J|_XKY}Hw4bh3u2M48Ty)I61g0B zWj&|QMR@h}^Qe4$x1h07w1wp(6&vN>&IT)fi%VYJl27+(kfcbIv_{hO-;3Gf!>r|F z(U~2(;aBaCW##3bAjs2~tqvb42CmNy(aM(^K7+vs+&=uFp(TQZEUZQ(hzL3Cugc$~ zXCgd1$bU}_G{Y=BNNqX$DiNbPr_IZe&~t0^QqOZLbzEXAbK>CoTC4NS+UzAh%?XT_ z+BM;3Y9c!Q@{=@4Z*bCodAR(_cdPamQ-28(6J9qui&loc2+aEz5)7B9#%YC|60YI= zOT6?Rjvoj(j#B<*G_m5 zgVJz-)iVDlbSRY~ZF-%Df4ijAejdlReUKoo+jG><@pDw+EdkAqNMc+*+rt?^d9Mou zJRMMh_nx3g$8_?i!3M z%wAW-c~;%1&&|#eKDWC@^nuUL$~v==tv+ejR#n|PzX(Oe=T%iT1r}p|b0zAyEZ|oU zS*+8Dayg_2^k=_lHIe2}exHPkM)n7*v+l;$rb|Ul#kcIr$(LCX0&;G+M6Ee#6GF~V zh|s%jde43dU4U9PL8MK-x_@yehW1r(Q-B^jH3I{~>0RGWs4X6L|3v}& zhy)l(TrGLZh+Ew}o!UA_uKd}uIP48fJqqURG$-@LGYvhXJu z6jQw%9_?AHXxUHNuw2LV#cT8)i@$N0a|E}eCQ6enw8tdI%%>#ibNg25`#CcBt99GK zI8@DWYV51IoD=~3xeHJ z4HmqkvBX*Iqpbr+9E=gXV=$z+R}d@sr=#_ShxU&^oFge}|6E^XN5|TlZ89N0d6%mw z6@Lx%>1(f!IBi76KI}9XRLG~*u)D@T5%A7EPhVX6QUbB8T*FoP&gqVAbaD8V6i5Fo!zLAYh&5Q8LeeE)@(CRXi-{+PU3>Yad z?tLwK7ne^`J4bbD4Y6p{z*KXuF0v9{%F>U8r*A7AgmdMwBEm5=%8@+XuNcW9zC?87 zF|zh{srb^_Zp2a_naSzJ)auj^tQa zxS?ypnc=qW4+{M$X<9s7MPXt zIBwnVljnQSr5)jxSd(lJ^2?DJ0|~6%K78{cn)nnPY+NL@uXd6wQpT9@Dpu(a3o}jH z(W#tb?FA4OuNYr$Sx2`|bIndCoDRFYPApoq>KtFtTf|E*5_*&-yJzMWt80sKhe>cI zh7z!bJukcriW6VZFHiSiz$cCUH@5#$Ix@1B)mhACz`B@r5ar!b69ztp`M)a8qJi{U z6q%Z~PQmes{kY-#c;iDsaywOZz{KW1dz;y+L!3=$1Cw@%_20g6W zaf3{v)Bd>MCa%dRHu`0y03zYv8zJaK;6M>9Eq>#WeKC_y6%%R(I+-34iVMHVrzUUA zm5nl|HrBTtlos~F(|Zfb7|576-rA^uF>!9a_bASo5d!lKJi*eE5&2)Ue5|I`zLcBC zK;#tiHc8;1m_Tqkircylst-evP=p)D8E$XO(azyUw?0QOyuo1&lPsdI7@49_a3TDe z-1^M&`U#h+yhQ#tYp@fsRRi}gbS3E+DqkOuj>_z1)Q7k~)NAqb_r0zJuNCKE)vP0t znTGZa&q~l>-E*HrOS@kLSDzwPg(?opC*iDskynlYvTZ~}1Ze$oG3K(#0{NK3)gn$E za-sxTPi0FHaKVH{b|gn64XwV@oL(wP?kRTo-*W%D@>aDv!0p)y{sN=0%mWOl)B~FG zch8L9Fev!s+Ts6#Nq{L&XR6bXL>O?pByelWFTR6p14?mOts1*=sF+&^)D+>yW#XJs z56+&-1fN8lf19VSqZ#`;_{HxFlB7dM+~)GpN!h)o*5EZSMF!OLa0LnDg>YT2f^5e9 zs97HZpZkMiVqP4)qjRp_F|f!ZhO7D5=q2hA-e%i#F;h z>alH1iXtj|>!_}NC;lde#O!^W5UM494RKz4ParfFY_xbm8jXnS<-73n9``dM8y%l` zi$VcXA@(g%mQLV*FJR2%x?W>CiWfutirbkaGeKSkKE>)t=H{Mn%x5NyX)M5tfP}f) zEalq5V7IH_a3+j;WL#?RCz_;N|FRlKJLQmbn3=C zd!SKH|Ij0^NSRCLvh@(@)Hj5u$!vqLa5sI~mgX6G_51Kb{t=}tZkTc_I=_rNvn#phQ^i1{u~wE{^zC8`_XD4wGA1*^dZZMh!5owy#U{ zDyU5clv8Zj&4dXed?pPcF!pl856bI&;>C5JO8{Nzyl*lyZ--wMss_1N^9H?}HyenVA%Pu-P9M z-1Icz{7Aq2!v6|hG*@1v96{4h?LERl+?)I2&$(UFI(zKvq$@0PzF?gSLaD5cUH!X; zXFuy8Shg@OQY(WaE=Po(TMfp4j%zBmqr&K262JXkaXfSWab5K|gO??I3#(SC_15X^ z2vIPxxgF=|ix5JW;`(bfi|=>;@?unTNvqALRz5*Q5RkvV`3mET}1P22yqmcg0KH z)zwSN%Cs-Z*a=^b5#>N?P^4b5@$uZNzONjTp&2&|#Kek$Qn1fAvm`1Dk>-5HA-fJw zbG?QulKV8MzRR$xMJ4AZlWW{G(5|)C5;&>6KAa zS6VAH1=-rn89vGfYPbXfbgI8|jvd-QDF3tnv|=>p{Fi%THfmI_-VFT&>b%Krs4KF_ z!S?tP%V)1sWAFv++>7tHVkX8x{-}e5y8TF|#jNz)2NjPP;z})E6Bc;{cQ0TCby{h+ z%W1NiPb%s64AOYY-7Nfg7qb7IaK6$PGX3?HY0Em6)VkNTuj1h62PzJ-0B{Ox{&_hJ zgqXhjj#=EB{G2`u&kBPK5*+&vqUumvzLbjwZ}QH`Ib0FT*w(Czt#jY^r9vo5D@xR? zhCkE0YE$F0o_D@mr?(z|>&N>p-2mJh55}$utvH(FiF70MvIRovsnMWzw9bb3^$r;M z(QL4Yvp~G7>S!#oz*K31Go`h2Xw|9EdOIO56%lKAoq8C?4}FzGDl4tTcSp|=@FcKw z+!K8A6aVs;nLsWo*CVN{Z{PZQ=J?H`v}evq$(&6hD>Zi#+R#3nL+|j**E@rMss6gD z=Y~uy#dfoOIdpNfTmk;u|5bL@Us-(78mCjbM5K}KE=>{o5L_$Ei zd8NC%yHmOw?%{jyKX6&g#Sb#`&YXGYoPD0X_viWPecW9lPjX0N{a$fcY*U|3F_E2U zoCd{sqpj=7ovrKDREg%CFdGkFOo~Mmv^9D#7+lV2mI3>DI$T39Ht%iKghQVu-sELu z-?>L5pa8pXLErj>L83$v_!?JeWdF1zpKi;PG{$7JOwY_eh9NKSxShnP#I!whp!OUA zbL|70wAZilXqE9nz^G_${<^{#lzv6}L?yCsA%?PGPWD|8Oy;DLM47G;$IEa563mGP zOX_Cvf@gG1l?$!@Wtf5Xt~ee0yl@%SH-~j&Gt^K7ahNlH_p^RB&{cai znsz_tQBHYD(A-fERt3c5)FycY+*@Dq$2Q#TF!PP1xnR-gm$L7s>J;sUvtLaAy$L)S zzIYv)WthWxQ72(iAD&Q!O$-&WK2s~5Hps~HZt&y~gJ3LO#zQBWWR@|69KNu^nYZDS zl*ZRXycVj%KS|m2VQPfQHOSk&2^l_-O9(A9-UV+fL$m3fUetJOVnub$kmlu;8Fyow z$P)Vf)x`X3^7eHxbbw6D)7?)fn`<}yPR42e@B7>O#v|PK+E^_YPKhGrGb7|+t_$#O4EaiJXZ5bCA?9)bz!QHJG zszlPA-Fj9q@2aL7HElI?vYKMS!_QDU*}(BDPhYHFEs1gE==#!qe8Hu&2YdMhRTpNS z #79pZ2q;;?qND4`k3h0Y|JGt_%yT9JqT_D$a=EZdQyDlEQl8V4f@#8}7cwiq$ z!H%Dr#0a4g1|*8jwUo*nN!TcuKEGIW;+j61FuZEW(#0f@3H7O1mnGA6S(@8*JiG7E zaHTBRov;P!=E9YYfVC5`eeJP#KW^iEty>t4%nxdoSy~4(h-qdMpPYuI8!*yjTwL5w z<1(7ULnMc#t;otkJ6soT-7`yO1&~NIDH~4kHOvM@pB(0OfAwFs&BL&*Cc>FAPu5&M z%dac5LL_G&WXkc_sUj)QEDO1sOp}#Ss5|_mlJgBLyy{vfcScS09+=>TAi45WVL1T& z-Q-F+l=n~$=Lg-lZ4`ZtI+Vui35YE|7@Q59X*nB4uMK1ODjHjhTYfOL4q>vkVkf3V zO^}P`ppJEnm;nPvhs0uQ`@N<0s4BtQ5AHX@{xV# zvgf?g>rU1=2Q0Lf{Njcgk2hS0W?#fUfPx2?1e+zVrlTR;;>q5t&9p(#{@-F~$CYAL zJ_~))zrLv{2R4USGrZWj)437`r+jzP)yljEI6$(!i2%4LqJ&siQtx^ zX2Gpv?P+O^ae8D`IRdFA)w?J+)b9&FT#IhOw{=7gBbp8ftkC)ZLY^s6H3;Sds z8IjRhG+=B!n2kP=`a9_CzGnh4lLNsgkITh4N^WUmubSt zg6n?ujFa)Sn5klM=xf?Md>`sPd*S{|zd#PF4E>S0K7B(;(fsq7f6*FTmg9VqN^`r}M`A+EIInWKwy#Loo+UBrDC+v)Gt-OFp?#CafDk`pKgt41~?)Bpm@&QH2l(nV*b!)!S*3iq3>umd(-`e4#9&gFAub@d=w_v$tMbHm$0*A1z=7kK+Rol z4I&^TQ}p!}1C5-(WvkDk+#)V?@c@_>`GafgEHJEpyJLBWdMLnz1B4MM{(IB$(Ac3M za6YiypZmM4x#-u!Q_Qg*5g$V!@4O9rzF&Jn);3*1LIHRQy=>KU5}*eZ(6u+eV6szj zap9#FIXG8pX}@=nC%R~?ND77j>HRIue6_T;vAOF@0gtfJ=U0BOsQg-&e-#!+L`Gt3 zX}245{u)KZ zIDj2)W-2ZIWk>R`=k>tVm-3@8FZDFX4t%yK5gmU07lcZ zo&r$xy}d<2FE*%Ni~aFK3DEoaQ-HhkU8Q3hJMd%ZxCT4cLITg%6V&%{-emW(ma=HDcx7&FgL*@qK_k#CjrG#Vm0f3nlx3XdiVymN@JHhAJps^qH(p%DDTX(x2Ynolz%$nRH7UafZ z++J^3_kGupt0=7h_fdFeW~PGpH3?Oi<{2~hS~2K%5|rX(T3)1eYeoP}Gct?M&#*l0 zVeT}Pc1)uMC6kkDbh;i(or$sW*#^{mECMPGK+H(apAtV;@TOTpDyIm9PSv&Rs`M3(~BbyHf z-)~6mZ9HL53}g7|{srKh|5Y@sPBz~2iQh~Qa;CIX+m3g>g&jaws=7YE60oDv0$p)- zE-2qaM|cC2-#DIp3Ja?onwjd`M=M$!A?TBNdEslz@5Ht2v|EVH(0JD?%R2oD=}k`Q z^653$iS`ENpKItCex2NbCk#g+|Lphd_=7FxU-4fnC&-8HKCqXD$!|p%3q?B`VrlK> z^!mt*>{ibPtGOAW`GM#;E#*laWo4|31+U`V@de=}WF}NF<715>Jiy)Wv_&}7pAiFx zjp~4atibggm);+Eb#>;0dMN|<8!4knB9&?2^yEs*^zajr@@{E?wJ$_t7=>hfo>1#G;dwxpm#?h$0~o5&B~EPrFTq zNu}+RfZq=N&wB(KxwNNr1>C>pv+9-kFWznl!1P~2etoGTZ|*k5AO4VWnfeEQ%07nn zSNiK3mxq|dZMy^SQfq?|am~uPt2tl(Lk-E3$djop$?Q!2W4D_M`zVcmm!A3~U6YXg zd20K^-Q?Mv>sh!a#yOB00ij0>m=d6iur67DA>sr{xy(SgLy|9N0a2qp*L33J`md!5Ruj*MfSItU+RhWr}2h>cO?`|kzE2Fx*b`}#8-FmKhIeFuHfmLN#b z2YwRaWCD?WeMOXlsJ%AsyJy<&HGh}>onIfJBD8cN&%+&B=D#Y?zJ)ux@oquYDb^)q zZcD=IW9nPV6UzL~3wd(tlpn<%oZQ?2x_rz500_{C^CO)QQbg-i5$;(vFg##*CmL5r zee?GM?3K3m_JDe8IYVLbdJFEThVs(MYzd}P3tzX+7s;u$@W95gZ_xLgr>Ccf*XX43 z6%K`HlKeONn1low;G9(*Yw_>dTaVN6-(yvu7S}bRzXz*Bv|4K9RMb$ieICL6EE(c3 zrJ8)n_ar{i2&T*vP_>t~drB7`xF5lt*xTL`fO)m83Msg_luz_7Ac!SYH-Ib5e&UV{ zduWRQOgk}w`HuvP*Avw(l5j);ESoCEH=i@TOUsY@ILZt=CB^HaTbgHd9DRIV0^AH` zOBuOkAC%M#Vv3e%YRRV}uB-qa3GVPrP`SS0PlT-D{g-*|{(0Xh74^%7_-_Hk0P)|a zhXNpyh+WLN1cJFZBR}T^eo3r}#lZ`x&CmcQ(rVR-%G@Pj0K42@ZEC8i;hv+Uw71ggnv`m6GNC1(j%Hy>LHaR_El z{9cwU*Vd|IdO4%vH)2q)SDjD=D9Ep&Br_ zc>+u{GtL@r_7y?fzUG@R^GLVBO9*vRp{`?^h@vVbDV4|TR}>t`R!&LcG(iAfI)tI-p3b6QM^<#}^PdzqTHg2Xuk{_^ zl-P~>D%fO*NY2=ELCz-utHI*Z?gpFFqm%1eU|zE|-4pjbwdw_m$b@4nmOxhV+e+r< z&67`O+w4n*U+$=0jEqpI!TcU%ODq@UnXfy=?_&TOSyNY+vF`Pz66pZ6Lr8IbZ(1sDL!JN1FmugRX3DOt8l)(ltE| zOYX{?rf*lg5`UCYX5{6$dk;6_k~(mia;02Djy3uK%!2>GkmkJt9d|RBkv)5cUn`Ui z!PkY`EwKQlyzH4f`Q>s588D@32WY}huy6|iUM6x&A*byK=TF7p^~Q%IBCi_VzEKto z@jB?^lWHTL`d(t~$IT6RKtk9-U%mGK24S+pe$9R<%wvuy1QIsChK71)6{7YoOYx?f zWXVti!YS`FRrRHF58A8cv-02_nN2q$q60YHV5Krr)oFzSj>{T#34n_SvG^LG>*x@( z>KdPV)_2(ZN?|rf?l|E0$w$i#Q$$0TeyE?r{B8-{>K^WI9*MjFY>pGwPD8jKBU$Et zJ}FRUwXL)8PFt$5)kXB(P}Zb+lJ%yzebRu3878k!GaOiUmfad}$ZOpWP>canI!S3} z;^!dcuRoQkBca0B&xOy_>Q_ZY98k&1-H%K`U$Q(PIjeFQ=ye_HQnT3sy$m6qTAb zE|-TUVBSBxQDqq&L9L^74$$}n%02?N^6f#C9 zKb5sHeLm`W_#048MSE*9Jc=bOAALW(9iR^*5#^zBcrT8DRzOCU*Qc-tKu$`r6V%l~e<^o%4 z5aUCgk>r_mbw2<;*)9IoxrPuv%Dn_h^k>fH)3mQB~2`9u}{jebC_u()`$mqXRQ z5C#fU&iCiNs-P9IFPIUQ+*w0xcv>r|RB=JFR=rxrV`_f@C>jVsFIY z&I?>;y{#vnplrFcl)p@4Rlcm#n45Iyf=WII;FTW~En@Thc{J&%vYT!JacT=Nz4T9TeB)kk2$*(l z0BMJszy`g`zN!J`p%fl#c#l>9z&T0f>aVbLvnjVJ7imb;jt`3kMKuL8BP!uNo%9RGh{<5XTi`svSAu!ye|#x+e`(fm{f`A4 z==<2d2@Kg(nHCI-6Guig9_?PhN5$^kS-rKqfav{-r`frAy~*BKacdE=E16oO;>G;W zNHNSdxJD9#Z$NDT;nd<0plvznv}qEsnl5+62YJ?fox&B6mRwnldN80Hu;R55I~Ze(;x8xt zL^svW6Ca3@esh^f?L>}3i0~argmzkDavT@m5T|+USmsxxah5EpBd@voq(AU!wo)8u zLhxk$xOzts0UaZN2^VO3N#Nhpu=W-}rj_;e<(#Vp=vQF#1f2Xbwvnk}*AKYAzVp;O zEQ$dZCw)=<`P+TiDmDmOKXr(mY3fY?c-{(-cTj;jtY8j%&Ad$uyJJ*tUg#*(*ce-r z%u*YXLI4yL`A9%LJwl&yi@DWrJiRtkI>H4L3^sFhL_S?^w6o+ zPOnt10>YCm5y z8bKCmGZMS%A0t2!B}*7M9qImB4NGqVWQk&Z@%JqyHK#ZIb`3B4?wWA?AguHUDD;=Z zsV~*sV~g0H|2-LP(J=fXFY z8R(%>($dIa4lx52fvkpgNkBmXKI#M8VW}w=vTKFTfAuql8i>sagFs{?g%3xJSAcQ( z)Dg>3m?&Vb31P_%Lh||E88JWfbs6)^__RGjtm>Hw*v$6yD1h2lfEZM5IgffS6dN8M zzREQYg-^TgsWt#Ko<(APT-# zQYf^Oi;~^ivywPBhyxo{47ct?q)72MhJxx}DFB`JwQIpKJ=r z4jVGyp^c`%w?}O*VRw^TQx(82LY$(w_WEIrwjyUNvE@Lc_KLHv&lyF60!{gB5)q=^ z${x0}&9t^=k*f5P3X=FD(|?%9Y{Hg};Xx(`N*ySfn+5eb9AP+(or;|3Wf(tWgZDKFvdq0 z_Q0USlSl}3MnG-*aDr5W)y~wNp;dVj z`afLPf}389$vB$mK3-l3#MqKYcjYZMXJ4&W!Ir(HAA6L~%p~vB;-jproYBuc2ikYu z;gcs+P3{ndRv{#1jE!K!eeX6M25T$H2ljYL$;bcaOQ@20DJ5vtt#;0ag}hHRP5tMCfM|QM56#oG>eEj={s|3!x|dY` zNSa}BRL=q*^0sT=vwQc~rV*6eqyODZ*zdiQD_-eYD^A>qywvZ{!(MYLTG`qXut{Bm zqaRNYU!pCT*!z2%f-5CLJ!d)(1N~@4x(iV4W2{8b&p*Ze_bHLYF;=#EGyGvp|B`R6 zA?^4+EvMEsumn9LBNK5bdR0qb1D?3l$U%lrEJp3TJy7RBrpPAl`%GR2SeuFWs;c1|rxbX_cb1=zZ|;(%Gfh8a=W3S6JtZIxgIS*3-`z-ez75A zxI7@N3KUjvpfWqkVPl36evqdNbNyADQ8>EIMV4ucmP0okFn$NFUI;j-Zn3>^*8T53 z-piqq8`d#+#BFIP8DKlet1=A!|n%uu=^ zI2}!h)i6|)pPyM-$>)8@jKRf~0IEFukjUiE9TnfvIwKi3KF5C?;#|TH5gTw*t9I%x zj*=%D{xP&5G6en;vZo0&LUfPgrp_<}gzaQ>e$wvq-0sjHoUEEK;IEVl3=1+&EJ(^C zs~%@X479KV7va?81Xo`f2`#M+&h1aon#FI73vDf}rHi0(|2!GBSBn3=7rg!x??K`0 zVLSSHQm$Nyd{}vO+6(N9*;=<^s}nqPRM*EljSF^lAI~b|F12E4i!}|qMT=FW$H8}v za{H@qH(RBAnqbh?h4=V}gpN{lIsmc<3IdtXOV@sPNZ-CzCKq`stp+i)FFPKBY`@}Z zLccTPf3M1C>|c2&9fj|zM@I1XSH_tzK)P1u6n>qqozv4Q!Ev$3pw`OXo;dZZ!T%iq zFM!Z^4hl{JN@w6UBgn&=T2|#3XB>JZL6GKVIx0Y4eye4=JW0I zz1aYVr%*!Lz*xF9o(E@$4#7!w1YwN~_$Gm+_KlRo?9;NU7mwo~EQ@>|;tEO~6p{w) zhj#uRwUpwKT#=|Pqb5dDvncCNI{*Z?mX<%?+Y7|B9uum^?>oI~Fh58dm1iip!zod` z3|7}W!uy2#?U>-N%>t51CQ49ib1z4u5=4coGgg9kseldt-=#Up^}a0ezH*LYw#s^H zTYgyYpcI;PZgxbBc;V~C9&-K^6la0s2>0GT8SAC~-%pCfD{v&~;29Ln+yT|0MREhl zyx|_SJlA)fDOke3UbA)`PkQn89YEsH0lFcxudY2|r#x&tK7O1tpo|FqS4s3R=)w06 zbf(~ho=+E8_J|Iz5yApR|@fCeH3Psj>%k{(9d6t@)p` zv$Iq~yq3@U%@(3Dk)h6M4Z2dzfJIpY;p>{(j3#XthQqrs8Kl^j=mfjTk3p|W49~$+Y6U@EIMpHY(u*~O zEGR35@t9DKCh6&yFc-6mf+O&?Z{V{Y$)ZF|wy>A}gh=UInHsR8XuW^8OVN-rCix9Y?&B{n2IJ9U6OoWK z7U$)u?x7f_8SOdNjCWWBz17h!reH1B4rBQ@mN!FR*}tqMqHhlhE+%9lbifM#2XgXb ze>8LQ3~u*{m&lHf73g9=cQO7dVp#_8b^Etq14ZAsc%XiRUOiiAzfh-l*No%)E4hC| z6a5^^Ih!R$4uyM>ermUhh3m8d$Y|rhE^?rpX#)63&ewUAUjx3R{>%1*n;NI1Psh%W zCy&IwByS!mG{AsXfdSN--e2e6o$JnL-_=08P+3=}?*sNLV<2HsFJfAVAl>xQgXsl+ z_unt@?%~Fa#)$}jMz_b(Zr2$8fil^#$jC?@;%(6JSTk>53Cz3=K-rrx@G__@DJiKn z-9x=cH}6qo6w=I6RmP8KxqG*S)5O#&1Y-E_07nchUJbZ0va*`D1>+_4TYwa(a4I{i z7Z)zt^M@06(dS~`N&o&m?hN|D+d16`iGwPE`MW~q=8DrBzbMIaq!94^T>Bx3GlzYP z6FoOCpjW5^|ep{)WMlKv1Qwp&_BENwBvE?Y))f zGEhgUL%YSYN=Q_qiIH0pH705{RkT66MMk?WgwV6PdTC?i=$6}X3nJglM48TGc%$j# z&~<@@LXIwno)i{jJV^#ohE?EL(QM^35OlrRvQ3n`eh$KxC%mTU6UUeAoIK`N$peQx z^RGmjrQ7+MoQ=ts&|Xr};WI%q`0^aMszV1a(VwD*({0&S!EX?>-R_i#GBhv5foRkC z9In>yE0RcCZ-xq(i}Abl=t)ZmP6OVl;C|p}=C@=+@);BJ$fLs#k$SwI|L-rU|Hl!r cpSvGWzkFupXLFcrg#$nG(kfC#62@Qt2ch#Iw*UYD From 2bef0085504c17f5c5185c64e9219aefcde18e3a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 14:51:34 -0800 Subject: [PATCH 010/551] smaller --- README.md | 2 +- pgcat2.png => pgcat3.png | Bin 2 files changed, 1 insertion(+), 1 deletion(-) rename pgcat2.png => pgcat3.png (100%) diff --git a/README.md b/README.md index 58c66e469..85f1e1225 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![CircleCI](https://circleci.com/gh/levkk/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/levkk/pgcat/tree/main) -![PgCat](./pgcat2.png) +![PgCat](./pgcat3.png) Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover support. diff --git a/pgcat2.png b/pgcat3.png similarity index 100% rename from pgcat2.png rename to pgcat3.png From 1f742a17ff378611c7d0893c16bbb8cd2913ca75 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 15:23:21 -0800 Subject: [PATCH 011/551] server version \d+ hack --- src/client.rs | 13 +++++++++++-- src/messages.rs | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/client.rs b/src/client.rs index 0fe185983..4accc8e49 100644 --- a/src/client.rs +++ b/src/client.rs @@ -20,6 +20,7 @@ pub struct Client { buffer: BytesMut, name: String, cancel_mode: bool, + transaction_mode: bool, process_id: i32, secret_key: i32, client_server_map: ClientServerMap, @@ -89,6 +90,7 @@ impl Client { buffer: BytesMut::with_capacity(8196), name: name, cancel_mode: false, + transaction_mode: true, process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, @@ -108,6 +110,7 @@ impl Client { buffer: BytesMut::with_capacity(8196), name: String::from("cancel_mode"), cancel_mode: true, + transaction_mode: true, process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, @@ -200,7 +203,7 @@ impl Client { } // Release server - if !server.in_transaction() { + if !server.in_transaction() && self.transaction_mode { break; } } @@ -253,7 +256,7 @@ impl Client { } // Release server - if !server.in_transaction() { + if !server.in_transaction() && self.transaction_mode { break; } } @@ -276,6 +279,12 @@ impl Client { return Err(err); } }; + + // Release the server + if !server.in_transaction() && self.transaction_mode { + println!("Releasing after copy done"); + break; + } } _ => { diff --git a/src/messages.rs b/src/messages.rs index 7c076695d..3f914871e 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -6,6 +6,13 @@ use tokio::net::TcpStream; use crate::errors::Error; +// This is a funny one. `psql` parses this to figure out which +// queries to send when using shortcuts, e.g. \d+. +// +// TODO: Actually get the version from the server itself. +// +const SERVER_VESION: &str = "12.9 (Ubuntu 12.9-0ubuntu0.20.04.1)"; + pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { let mut auth_ok = BytesMut::with_capacity(9); @@ -18,6 +25,10 @@ pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { pub async fn server_parameters(stream: &mut TcpStream) -> Result<(), Error> { let client_encoding = BytesMut::from(&b"client_encoding\0UTF8\0"[..]); + let server_version = + BytesMut::from(&format!("server_version\0{}\0", SERVER_VESION).as_bytes()[..]); + + // Client encoding let len = client_encoding.len() as i32 + 4; // TODO: add more parameters here let mut res = BytesMut::with_capacity(len as usize + 1); @@ -25,6 +36,11 @@ pub async fn server_parameters(stream: &mut TcpStream) -> Result<(), Error> { res.put_i32(len); res.put_slice(&client_encoding[..]); + let len = server_version.len() as i32 + 4; + res.put_u8(b'S'); + res.put_i32(len); + res.put_slice(&server_version[..]); + Ok(write_all(stream, res).await?) } From 0578b9bf135571bfd4a4f4ceefc4d7a80ef9d935 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 15:33:10 -0800 Subject: [PATCH 012/551] :+1: --- README.md | 2 +- src/messages.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 85f1e1225..86244530e 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ pgbench -t 1000 -p 5433 -h 127.0.0.1 --protocol extended ## Features 1. Session mode. -2. Transaction mode (basic). +2. Transaction mode. 3. `COPY` protocol support. 4. Query cancellation. 5. Round-robin load balancing of replicas. diff --git a/src/messages.rs b/src/messages.rs index 3f914871e..9a74c7f70 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -30,7 +30,7 @@ pub async fn server_parameters(stream: &mut TcpStream) -> Result<(), Error> { // Client encoding let len = client_encoding.len() as i32 + 4; // TODO: add more parameters here - let mut res = BytesMut::with_capacity(len as usize + 1); + let mut res = BytesMut::with_capacity(64); res.put_u8(b'S'); res.put_i32(len); From f1739b1268584e75376dce9a93e533e5001753dd Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 18:20:53 -0800 Subject: [PATCH 013/551] correct load balancing --- src/client.rs | 51 ++++++---- src/main.rs | 18 +--- src/pool.rs | 257 +++++++++++++++++++++++++++++--------------------- 3 files changed, 184 insertions(+), 142 deletions(-) diff --git a/src/client.rs b/src/client.rs index 4accc8e49..5b0e21800 100644 --- a/src/client.rs +++ b/src/client.rs @@ -9,9 +9,8 @@ use tokio::net::TcpStream; use crate::errors::Error; use crate::messages::*; -use crate::pool::{ClientServerMap, ServerPool}; +use crate::pool::{ClientServerMap, ConnectionPool}; use crate::server::Server; -use bb8::Pool; /// The client state. pub struct Client { @@ -125,7 +124,7 @@ impl Client { } /// Client loop. We handle all messages between the client and the database here. - pub async fn handle(&mut self, pool: Pool) -> Result<(), Error> { + pub async fn handle(&mut self, pool: ConnectionPool) -> Result<(), Error> { // Special: cancelling existing running query if self.cancel_mode { let (process_id, secret_key, address, port) = { @@ -148,13 +147,17 @@ impl Client { loop { // Only grab a connection once we have some traffic on the socket // TODO: this is not the most optimal way to share servers. - let mut peek_buf = vec![0u8; 2]; + // let mut peek_buf = vec![0u8; 2]; - match self.read.get_mut().peek(&mut peek_buf).await { - Ok(_) => (), - Err(_) => return Err(Error::ClientDisconnected), - }; - let mut proxy = pool.get().await.unwrap(); + // match self.read.get_mut().peek(&mut peek_buf).await { + // Ok(_) => (), + // Err(_) => return Err(Error::ClientDisconnected), + // }; + let message = read_message(&mut self.read).await?; + + self.buffer.put(message); + + let mut proxy = pool.get(None).await.unwrap().0; let server = &mut *proxy; // TODO: maybe don't do this, I don't think it's useful. @@ -164,18 +167,28 @@ impl Client { server.claim(self.process_id, self.secret_key); loop { - let mut message = match read_message(&mut self.read).await { - Ok(message) => message, - Err(err) => { - if server.in_transaction() { - // TODO: this is what PgBouncer does - // which leads to connection thrashing. - // - // I think we could issue a ROLLBACK here instead. - server.mark_bad(); + let mut message = match self.buffer.len() { + 0 => { + match read_message(&mut self.read).await { + Ok(message) => message, + Err(err) => { + if server.in_transaction() { + // TODO: this is what PgBouncer does + // which leads to connection thrashing. + // + // I think we could issue a ROLLBACK here instead. + server.mark_bad(); + } + + return Err(err); + } } + } - return Err(err); + _ => { + let message = self.buffer.clone(); + self.buffer.clear(); + message } }; diff --git a/src/main.rs b/src/main.rs index 37d59338e..ed9e738c9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,6 @@ extern crate tokio; use tokio::net::TcpListener; -use bb8::Pool; use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -35,12 +34,7 @@ mod server; // Support for query cancellation: this maps our process_ids and // secret keys to the backend's. use config::{Address, User}; -use pool::{ClientServerMap, ReplicaPool, ServerPool}; - -// -// Poor man's config -// -const POOL_SIZE: u32 = 15; +use pool::{ClientServerMap, ConnectionPool}; /// Main! #[tokio::main] @@ -71,7 +65,6 @@ async fn main() { port: "5432".to_string(), }, ]; - let num_addresses = addresses.len() as u32; let user = User { name: "lev".to_string(), @@ -80,9 +73,7 @@ async fn main() { let database = "lev"; - let replica_pool = ReplicaPool::new(addresses).await; - let manager = ServerPool::new(replica_pool, user, database, client_server_map.clone()); - + let pool = ConnectionPool::new(addresses, user, database, client_server_map.clone()).await; // We are round-robining, so ideally the replicas will be equally loaded. // Therefore, we are allocating number of replicas * pool size of connections. // However, if a replica dies, the remaining replicas will share the burden, @@ -91,11 +82,6 @@ async fn main() { // Note that failover in this case could bring down the remaining replicas, so // in certain situations, e.g. when replicas are running hot already, failover // is not at all desirable!! - let pool = Pool::builder() - .max_size(POOL_SIZE * num_addresses) - .build(manager) - .await - .unwrap(); loop { let pool = pool.clone(); diff --git a/src/pool.rs b/src/pool.rs index e68d6c80b..049fb2e37 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,6 +1,6 @@ /// Pooling and failover and banlist. use async_trait::async_trait; -use bb8::{ManageConnection, PooledConnection}; +use bb8::{ManageConnection, Pool, PooledConnection}; use chrono::naive::NaiveDateTime; use crate::config::{Address, User}; @@ -21,109 +21,116 @@ pub type ClientServerMap = Arc>, + addresses: Vec
, + round_robin: Counter, + banlist: BanList, } -impl ServerPool { - pub fn new( - replica_pool: ReplicaPool, +impl ConnectionPool { + pub async fn new( + addresses: Vec
, user: User, database: &str, client_server_map: ClientServerMap, - ) -> ServerPool { - ServerPool { - replica_pool: replica_pool, - user: user, - database: database.to_string(), - client_server_map: client_server_map, + ) -> ConnectionPool { + let mut databases = Vec::new(); + + for address in &addresses { + let manager = ServerPool::new( + address.clone(), + user.clone(), + database, + client_server_map.clone(), + ); + let pool = Pool::builder() + .max_size(POOL_SIZE) + .connection_timeout(std::time::Duration::from_millis(5000)) + .test_on_check_out(false) + .build(manager) + .await + .unwrap(); + + databases.push(pool); } - } -} -#[async_trait] -impl ManageConnection for ServerPool { - type Connection = Server; - type Error = Error; - - /// Attempts to create a new connection. - async fn connect(&self) -> Result { - println!(">> Getting new connection from the pool"); - let address = self.replica_pool.get(); - - match Server::startup( - &address.host, - &address.port, - &self.user.name, - &self.user.password, - &self.database, - self.client_server_map.clone(), - ) - .await - { - Ok(server) => { - self.replica_pool.unban(&address); - Ok(server) - } - Err(err) => { - self.replica_pool.ban(&address); - Err(err) - } + ConnectionPool { + databases: databases, + addresses: addresses, + round_robin: Arc::new(AtomicUsize::new(0)), + banlist: Arc::new(Mutex::new(HashMap::new())), } } - /// Determines if the connection is still connected to the database. - async fn is_valid(&self, conn: &mut PooledConnection<'_, Self>) -> Result<(), Self::Error> { - let server = &mut *conn; - - // Client disconnected before cleaning up - if server.in_transaction() { - return Err(Error::DirtyServer); - } - - // If this fails, the connection will be closed and another will be grabbed from the pool quietly :-). - // Failover, step 1, complete. - match tokio::time::timeout( - tokio::time::Duration::from_millis(1000), - server.query("SELECT 1"), - ) - .await - { - Ok(_) => Ok(()), - Err(_err) => { - println!(">> Unhealthy!"); - self.replica_pool.ban(&server.address()); - Err(Error::ServerTimeout) + /// Get a connection from the pool. Either round-robin or pick a specific one in case they are sharded. + pub async fn get( + &self, + index: Option, + ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { + match index { + // Asking for a specific database, must be sharded. + // No failover here. + Some(index) => { + assert!(index < self.databases.len()); + match self.databases[index].get().await { + Ok(conn) => Ok((conn, self.addresses[index].clone())), + Err(err) => { + println!(">> Shard {} down: {:?}", index, err); + Err(Error::ServerTimeout) + } + } } - } - } - /// Synchronously determine if the connection is no longer usable, if possible. - fn has_broken(&self, conn: &mut Self::Connection) -> bool { - conn.is_bad() - } -} - -/// A collection of addresses, which could either be a single primary, -/// many sharded primaries or replicas. -#[derive(Clone)] -pub struct ReplicaPool { - addresses: Vec
, - round_robin: Counter, - banlist: BanList, -} - -impl ReplicaPool { - /// Create a new replica pool. Addresses must be known in advance. - pub async fn new(addresses: Vec
) -> ReplicaPool { - ReplicaPool { - addresses: addresses, - round_robin: Arc::new(AtomicUsize::new(0)), - banlist: Arc::new(Mutex::new(HashMap::new())), + // Any database is fine, we're using round-robin here. + // Failover included if the server doesn't answer a health check. + None => { + loop { + let index = + self.round_robin.fetch_add(1, Ordering::SeqCst) % self.databases.len(); + let address = self.addresses[index].clone(); + + if self.is_banned(&address) { + continue; + } + + // Check if we can connect + let mut conn = match self.databases[index].get().await { + Ok(conn) => conn, + Err(err) => { + println!(">> Banning replica {}, error: {:?}", index, err); + self.ban(&address); + continue; + } + }; + + // Check if this server is alive with a health check + let server = &mut *conn; + + match tokio::time::timeout( + tokio::time::Duration::from_millis(1000), + server.query("SELECT 1"), + ) + .await + { + Ok(_) => return Ok((conn, address)), + Err(_) => { + println!( + ">> Banning replica {} because of failed health check", + index + ); + self.ban(&address); + continue; + } + } + } + } } } @@ -150,7 +157,7 @@ impl ReplicaPool { let mut guard = self.banlist.lock().unwrap(); // Everything is banned, nothig is banned - if guard.len() == self.addresses.len() { + if guard.len() == self.databases.len() { guard.clear(); drop(guard); println!(">> Unbanning all replicas."); @@ -173,22 +180,58 @@ impl ReplicaPool { None => false, } } +} - /// Get a replica to route the query to. - /// Will attempt to fetch a healthy replica. It will also - /// round-robin them for reasonably equal load. Round-robin is done - /// per transaction. - pub fn get(&self) -> Address { - loop { - // We'll never hit a 64-bit overflow right....right? :-) - let index = self.round_robin.fetch_add(1, Ordering::SeqCst) % self.addresses.len(); - - let address = &self.addresses[index]; - if !self.is_banned(address) { - return address.clone(); - } else { - continue; - } +pub struct ServerPool { + address: Address, + user: User, + database: String, + client_server_map: ClientServerMap, +} + +impl ServerPool { + pub fn new( + address: Address, + user: User, + database: &str, + client_server_map: ClientServerMap, + ) -> ServerPool { + ServerPool { + address: address, + user: user, + database: database.to_string(), + client_server_map: client_server_map, } } } + +#[async_trait] +impl ManageConnection for ServerPool { + type Connection = Server; + type Error = Error; + + /// Attempts to create a new connection. + async fn connect(&self) -> Result { + println!(">> Getting new connection from the pool"); + + Server::startup( + &self.address.host, + &self.address.port, + &self.user.name, + &self.user.password, + &self.database, + self.client_server_map.clone(), + ) + .await + } + + /// Determines if the connection is still connected to the database. + async fn is_valid(&self, _conn: &mut PooledConnection<'_, Self>) -> Result<(), Self::Error> { + Ok(()) + } + + /// Synchronously determine if the connection is no longer usable, if possible. + fn has_broken(&self, conn: &mut Self::Connection) -> bool { + conn.is_bad() + } +} From cd1653465b04efcccdcdc148687e1c9a3dbaa607 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 5 Feb 2022 19:43:48 -0800 Subject: [PATCH 014/551] sharding --- Cargo.lock | 21 +++++++++ Cargo.toml | 1 + src/main.rs | 3 +- src/pool.rs | 111 +++++++++++++++++++++++------------------------- src/sharding.rs | 33 ++++++++++++++ 5 files changed, 111 insertions(+), 58 deletions(-) create mode 100644 src/sharding.rs diff --git a/Cargo.lock b/Cargo.lock index 3bb02fada..8cf1dcbd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cpufeatures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +dependencies = [ + "libc", +] + [[package]] name = "crypto-common" version = "0.1.1" @@ -342,6 +351,7 @@ dependencies = [ "chrono", "md-5", "rand", + "sha-1", "tokio", ] @@ -400,6 +410,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sha-1" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" diff --git a/Cargo.toml b/Cargo.toml index 342489176..bc8034a05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,4 @@ bb8 = "*" async-trait = "*" rand = "*" chrono = "0.4" +sha-1 = "*" \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index ed9e738c9..e2e4dd50b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,6 +30,7 @@ mod errors; mod messages; mod pool; mod server; +mod sharding; // Support for query cancellation: this maps our process_ids and // secret keys to the backend's. @@ -41,7 +42,7 @@ use pool::{ClientServerMap, ConnectionPool}; async fn main() { println!("> Welcome to PgCat! Meow."); - let addr = "0.0.0.0:5433"; + let addr = "0.0.0.0:6432"; let listener = match TcpListener::bind(addr).await { Ok(sock) => sock, Err(err) => { diff --git a/src/pool.rs b/src/pool.rs index 049fb2e37..e4097b9b9 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -28,8 +28,8 @@ const POOL_SIZE: u32 = 15; #[derive(Clone)] pub struct ConnectionPool { - databases: Vec>, - addresses: Vec
, + databases: Vec>>, + addresses: Vec>, round_robin: Counter, banlist: BanList, } @@ -62,8 +62,8 @@ impl ConnectionPool { } ConnectionPool { - databases: databases, - addresses: addresses, + databases: vec![databases], + addresses: vec![addresses], round_robin: Arc::new(AtomicUsize::new(0)), banlist: Arc::new(Mutex::new(HashMap::new())), } @@ -72,63 +72,56 @@ impl ConnectionPool { /// Get a connection from the pool. Either round-robin or pick a specific one in case they are sharded. pub async fn get( &self, - index: Option, + shard: Option, ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { - match index { - // Asking for a specific database, must be sharded. - // No failover here. - Some(index) => { - assert!(index < self.databases.len()); - match self.databases[index].get().await { - Ok(conn) => Ok((conn, self.addresses[index].clone())), - Err(err) => { - println!(">> Shard {} down: {:?}", index, err); - Err(Error::ServerTimeout) - } + // Set this to false to gain ~3-4% speed. + let with_health_check = true; + + let shard = match shard { + Some(shard) => shard, + None => 0, // TODO: pick a shard at random + }; + + loop { + let index = + self.round_robin.fetch_add(1, Ordering::SeqCst) % self.databases[shard].len(); + let address = self.addresses[shard][index].clone(); + + if self.is_banned(&address) { + continue; + } + + // Check if we can connect + let mut conn = match self.databases[shard][index].get().await { + Ok(conn) => conn, + Err(err) => { + println!(">> Banning replica {}, error: {:?}", index, err); + self.ban(&address); + continue; } + }; + + if !with_health_check { + return Ok((conn, address)); } - // Any database is fine, we're using round-robin here. - // Failover included if the server doesn't answer a health check. - None => { - loop { - let index = - self.round_robin.fetch_add(1, Ordering::SeqCst) % self.databases.len(); - let address = self.addresses[index].clone(); - - if self.is_banned(&address) { - continue; - } - - // Check if we can connect - let mut conn = match self.databases[index].get().await { - Ok(conn) => conn, - Err(err) => { - println!(">> Banning replica {}, error: {:?}", index, err); - self.ban(&address); - continue; - } - }; - - // Check if this server is alive with a health check - let server = &mut *conn; - - match tokio::time::timeout( - tokio::time::Duration::from_millis(1000), - server.query("SELECT 1"), - ) - .await - { - Ok(_) => return Ok((conn, address)), - Err(_) => { - println!( - ">> Banning replica {} because of failed health check", - index - ); - self.ban(&address); - continue; - } - } + // // Check if this server is alive with a health check + let server = &mut *conn; + + match tokio::time::timeout( + tokio::time::Duration::from_millis(1000), + server.query("SELECT 1"), + ) + .await + { + Ok(_) => return Ok((conn, address)), + Err(_) => { + println!( + ">> Banning replica {} because of failed health check", + index + ); + self.ban(&address); + continue; } } } @@ -180,6 +173,10 @@ impl ConnectionPool { None => false, } } + + pub fn shards(&self) -> usize { + self.databases.len() + } } pub struct ServerPool { diff --git a/src/sharding.rs b/src/sharding.rs new file mode 100644 index 000000000..bf976e50f --- /dev/null +++ b/src/sharding.rs @@ -0,0 +1,33 @@ +use sha1::{Digest, Sha1}; + +pub struct Sharder { + shards: usize, +} + +impl Sharder { + pub fn new(shards: usize) -> Sharder { + Sharder { shards: shards } + } + + pub fn sha1(&self, key: &[u8]) -> usize { + let mut hasher = Sha1::new(); + hasher.update(key); + let result = hasher.finalize_reset(); + + let i = u32::from_le_bytes(result[result.len() - 4..result.len()].try_into().unwrap()); + i as usize % self.shards + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_sha1() { + let sharder = Sharder::new(12); + let key = b"1234"; + let shard = sharder.sha1(key); + assert_eq!(shard, 1); + } +} From 3b0f9075d361bf8ec6db39fb4709d88920e962e3 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 6 Feb 2022 10:48:14 -0800 Subject: [PATCH 015/551] fixed extended; comments; re-use conns with open transactions --- src/client.rs | 115 +++++++++++++++++++++++++++++--------------------- src/main.rs | 8 ---- 2 files changed, 66 insertions(+), 57 deletions(-) diff --git a/src/client.rs b/src/client.rs index 5b0e21800..5502a3e02 100644 --- a/src/client.rs +++ b/src/client.rs @@ -12,16 +12,32 @@ use crate::messages::*; use crate::pool::{ClientServerMap, ConnectionPool}; use crate::server::Server; -/// The client state. +/// The client state. One of these is created per client. pub struct Client { + // The reads are buffered (8K by default). read: BufReader, + + // We buffer the writes ourselves because we know the protocol + // better than a stock buffer. write: OwnedWriteHalf, + + // Internal buffer, where we place messages until we have to flush + // them to the backend. buffer: BytesMut, - name: String, + + // The client was started with the sole reason to cancel another running query. cancel_mode: bool, + + // In transaction mode, the connection is released after each transaction. + // Session mode has slightly higher throughput per client, but lower capacity. transaction_mode: bool, + + // For query cancellation, the client is given a random process ID and secret on startup. process_id: i32, secret_key: i32, + + // Clients are mapped to servers while they use them. This allows a client + // to connect and cancel a query. client_server_map: ClientServerMap, } @@ -75,19 +91,14 @@ impl Client { backend_key_data(&mut stream, process_id, secret_key).await?; ready_for_query(&mut stream).await?; + // Split the read and write streams + // so we can control buffering. let (read, write) = stream.into_split(); - let name: String = rand::thread_rng() - .sample_iter(&Alphanumeric) - .take(7) - .map(char::from) - .collect(); - return Ok(Client { read: BufReader::new(read), write: write, buffer: BytesMut::with_capacity(8196), - name: name, cancel_mode: false, transaction_mode: true, process_id: process_id, @@ -96,7 +107,7 @@ impl Client { }); } - // Cancel request + // Query cancel request. 80877102 => { let (read, write) = stream.into_split(); @@ -107,7 +118,6 @@ impl Client { read: BufReader::new(read), write: write, buffer: BytesMut::with_capacity(8196), - name: String::from("cancel_mode"), cancel_mode: true, transaction_mode: true, process_id: process_id, @@ -130,6 +140,7 @@ impl Client { let (process_id, secret_key, address, port) = { let guard = self.client_server_map.lock().unwrap(); match guard.get(&(self.process_id, self.secret_key)) { + // Drop the mutex as soon as possible. Some((process_id, secret_key, address, port)) => ( process_id.clone(), secret_key.clone(), @@ -145,51 +156,52 @@ impl Client { } loop { - // Only grab a connection once we have some traffic on the socket - // TODO: this is not the most optimal way to share servers. - // let mut peek_buf = vec![0u8; 2]; - - // match self.read.get_mut().peek(&mut peek_buf).await { - // Ok(_) => (), - // Err(_) => return Err(Error::ClientDisconnected), - // }; - let message = read_message(&mut self.read).await?; - - self.buffer.put(message); - - let mut proxy = pool.get(None).await.unwrap().0; + // Read a complete message from the client, which normally would be + // either a `Q` (query) or `P` (prepare, extended protocol). + // We can parse it here before grabbing a server from the pool, + // in case the client is sending some control messages, e.g. + // SET sharding_context.key = '1234'; + let mut message = read_message(&mut self.read).await?; + + // TODO: parse the message here. If it's part of our protocol, + // don't grab a server yet and continue loop. + + // The message is part of the regular protocol. + // self.buffer.put(message); + + // Grab a server from the pool. + // None = any shard + let connection = pool.get(None).await.unwrap(); + let mut proxy = connection.0; + let _address = connection.1; let server = &mut *proxy; - // TODO: maybe don't do this, I don't think it's useful. - server.set_name(&self.name).await?; - // Claim this server as mine for query cancellation. server.claim(self.process_id, self.secret_key); loop { - let mut message = match self.buffer.len() { - 0 => { - match read_message(&mut self.read).await { - Ok(message) => message, - Err(err) => { - if server.in_transaction() { - // TODO: this is what PgBouncer does - // which leads to connection thrashing. - // - // I think we could issue a ROLLBACK here instead. - server.mark_bad(); - } - - return Err(err); + // No messages in the buffer, read one. + let mut message = if message.len() == 0 { + match read_message(&mut self.read).await { + Ok(message) => message, + Err(err) => { + // Client disconnected without warning. + if server.in_transaction() { + // TODO: this is what PgBouncer does + // which leads to connection thrashing. + // + // I think we could issue a ROLLBACK here instead. + // server.mark_bad(); + server.query("ROLLBACK; DISCARD ALL;").await?; } - } - } - _ => { - let message = self.buffer.clone(); - self.buffer.clear(); - message + return Err(err); + } } + } else { + let msg = message.clone(); + message.clear(); + msg }; let original = message.clone(); // To be forwarded to the server @@ -222,9 +234,13 @@ impl Client { } 'X' => { - // Client closing + // Client closing. Rollback and clean up + // connection before releasing into the pool. + // Pgbouncer closes the connection which leads to + // connection thrashing when clients misbehave. + // This pool will protect the database. :salute: if server.in_transaction() { - server.query("ROLLBACK").await?; + server.query("ROLLBACK; DISCARD ALL;").await?; } return Ok(()); @@ -239,6 +255,7 @@ impl Client { self.buffer.put(&original[..]); } + // Describe 'D' => { self.buffer.put(&original[..]); } diff --git a/src/main.rs b/src/main.rs index e2e4dd50b..ba6f76b62 100644 --- a/src/main.rs +++ b/src/main.rs @@ -75,14 +75,6 @@ async fn main() { let database = "lev"; let pool = ConnectionPool::new(addresses, user, database, client_server_map.clone()).await; - // We are round-robining, so ideally the replicas will be equally loaded. - // Therefore, we are allocating number of replicas * pool size of connections. - // However, if a replica dies, the remaining replicas will share the burden, - // also equally. - // - // Note that failover in this case could bring down the remaining replicas, so - // in certain situations, e.g. when replicas are running hot already, failover - // is not at all desirable!! loop { let pool = pool.clone(); From 7ee0e75d9ccaab3500cc3b23445d6d4746b704d0 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 6 Feb 2022 11:13:12 -0800 Subject: [PATCH 016/551] fix banlist --- README.md | 90 +++++++++++++++++++++++++++++++++++++++++++---------- src/main.rs | 2 +- src/pool.rs | 42 ++++++++++++++----------- 3 files changed, 99 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 86244530e..577c25427 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,9 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su You can just PgBench to test your changes: ``` -pgbench -i -h 127.0.0.1 -p 5433 && \ -pgbench -t 1000 -p 5433 -h 127.0.0.1 --protocol simple && \ -pgbench -t 1000 -p 5433 -h 127.0.0.1 --protocol extended +pgbench -i -h 127.0.0.1 -p 6432 && \ +pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && \ +pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended ``` ## Features @@ -75,7 +75,7 @@ to make this choice :-). You can setup PgBench locally through PgCat: ``` -pgbench -h 127.0.0.1 -p 5433 -i +pgbench -h 127.0.0.1 -p 6432 -i ``` Coincidenly, this uses `COPY` so you can test if that works. @@ -83,7 +83,25 @@ Coincidenly, this uses `COPY` so you can test if that works. ### PgBouncer ``` -pgbench -h 127.0.0.1 -p 6432 --protocol extended -t 1000 +$ pgbench -i -h 127.0.0.1 -p 6432 && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended +dropping old tables... +creating tables... +generating data... +100000 of 100000 tuples (100%) done (elapsed 0.01 s, remaining 0.00 s) +vacuuming... +creating primary keys... +done. +starting vacuum...end. +transaction type: +scaling factor: 1 +query mode: simple +number of clients: 1 +number of threads: 1 +number of transactions per client: 1000 +number of transactions actually processed: 1000/1000 +latency average = 1.089 ms +tps = 918.687098 (including connections establishing) +tps = 918.847790 (excluding connections establishing) starting vacuum...end. transaction type: scaling factor: 1 @@ -92,15 +110,34 @@ number of clients: 1 number of threads: 1 number of transactions per client: 1000 number of transactions actually processed: 1000/1000 -latency average = 1.116 ms -tps = 895.900600 (including connections establishing) -tps = 896.115205 (excluding connections establishing) +latency average = 1.136 ms +tps = 880.622009 (including connections establishing) +tps = 880.769550 (excluding connections establishing) ``` ### PgCat + ``` -pgbench -h 127.0.0.1 -p 5433 --protocol extended -t 1000 +$ pgbench -i -h 127.0.0.1 -p 6432 && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended +dropping old tables... +creating tables... +generating data... +100000 of 100000 tuples (100%) done (elapsed 0.01 s, remaining 0.00 s) +vacuuming... +creating primary keys... +done. +starting vacuum...end. +transaction type: +scaling factor: 1 +query mode: simple +number of clients: 1 +number of threads: 1 +number of transactions per client: 1000 +number of transactions actually processed: 1000/1000 +latency average = 1.142 ms +tps = 875.645437 (including connections establishing) +tps = 875.799995 (excluding connections establishing) starting vacuum...end. transaction type: scaling factor: 1 @@ -109,15 +146,36 @@ number of clients: 1 number of threads: 1 number of transactions per client: 1000 number of transactions actually processed: 1000/1000 -latency average = 1.152 ms -tps = 867.761579 (including connections establishing) -tps = 867.881391 (excluding connections establishing) +latency average = 1.181 ms +tps = 846.539176 (including connections establishing) +tps = 846.713636 (excluding connections establishing) ``` ### Direct Postgres ``` -pgbench -h 127.0.0.1 -p 5432 --protocol extended -t 1000 +$ pgbench -i -h 127.0.0.1 -p 5432 && pgbench -t 1000 -p 5432 -h 127.0.0.1 --protocol simple && pgbench -t 1000 -p +5432 -h 127.0.0.1 --protocol extended +Password: +dropping old tables... +creating tables... +generating data... +100000 of 100000 tuples (100%) done (elapsed 0.01 s, remaining 0.00 s) +vacuuming... +creating primary keys... +done. +Password: +starting vacuum...end. +transaction type: +scaling factor: 1 +query mode: simple +number of clients: 1 +number of threads: 1 +number of transactions per client: 1000 +number of transactions actually processed: 1000/1000 +latency average = 0.902 ms +tps = 1109.014867 (including connections establishing) +tps = 1112.318595 (excluding connections establishing) Password: starting vacuum...end. transaction type: @@ -127,7 +185,7 @@ number of clients: 1 number of threads: 1 number of transactions per client: 1000 number of transactions actually processed: 1000/1000 -latency average = 0.944 ms -tps = 1059.007346 (including connections establishing) -tps = 1061.700877 (excluding connections establishing) +latency average = 0.931 ms +tps = 1074.017747 (including connections establishing) +tps = 1077.121752 (excluding connections establishing) ``` \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index ba6f76b62..dcab2b28b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -63,7 +63,7 @@ async fn main() { }, Address { host: "localhost".to_string(), - port: "5432".to_string(), + port: "5433".to_string(), }, ]; diff --git a/src/pool.rs b/src/pool.rs index e4097b9b9..5515267af 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -14,18 +14,23 @@ use std::sync::{ }; // Banlist: bad servers go in here. -pub type BanList = Arc>>; +pub type BanList = Arc>>>; pub type Counter = Arc; pub type ClientServerMap = Arc>>; // 60 seconds of ban time. // After that, the replica will be allowed to serve traffic again. const BAN_TIME: i64 = 60; -// -// Poor man's config -// + +// DB pool size (per actual database server) const POOL_SIZE: u32 = 15; +// 5 seconds to connect before we give up +const CONNECT_TIMEOUT: u64 = 5000; + +// How much time to give the server to answer a SELECT 1 query. +const HEALTHCHECK_TIMEOUT: u64 = 1000; + #[derive(Clone)] pub struct ConnectionPool { databases: Vec>>, @@ -52,7 +57,7 @@ impl ConnectionPool { ); let pool = Pool::builder() .max_size(POOL_SIZE) - .connection_timeout(std::time::Duration::from_millis(5000)) + .connection_timeout(std::time::Duration::from_millis(CONNECT_TIMEOUT)) .test_on_check_out(false) .build(manager) .await @@ -65,7 +70,7 @@ impl ConnectionPool { databases: vec![databases], addresses: vec![addresses], round_robin: Arc::new(AtomicUsize::new(0)), - banlist: Arc::new(Mutex::new(HashMap::new())), + banlist: Arc::new(Mutex::new(vec![HashMap::new()])), } } @@ -87,16 +92,17 @@ impl ConnectionPool { self.round_robin.fetch_add(1, Ordering::SeqCst) % self.databases[shard].len(); let address = self.addresses[shard][index].clone(); - if self.is_banned(&address) { + if self.is_banned(&address, shard) { continue; } // Check if we can connect + // TODO: implement query wait timeout, i.e. time to get a conn from the pool let mut conn = match self.databases[shard][index].get().await { Ok(conn) => conn, Err(err) => { println!(">> Banning replica {}, error: {:?}", index, err); - self.ban(&address); + self.ban(&address, shard); continue; } }; @@ -109,7 +115,7 @@ impl ConnectionPool { let server = &mut *conn; match tokio::time::timeout( - tokio::time::Duration::from_millis(1000), + tokio::time::Duration::from_millis(HEALTHCHECK_TIMEOUT), server.query("SELECT 1"), ) .await @@ -120,7 +126,7 @@ impl ConnectionPool { ">> Banning replica {} because of failed health check", index ); - self.ban(&address); + self.ban(&address, shard); continue; } } @@ -130,27 +136,27 @@ impl ConnectionPool { /// Ban an address (i.e. replica). It no longer will serve /// traffic for any new transactions. Existing transactions on that replica /// will finish successfully or error out to the clients. - pub fn ban(&self, address: &Address) { + pub fn ban(&self, address: &Address, shard: usize) { println!(">> Banning {:?}", address); let now = chrono::offset::Utc::now().naive_utc(); let mut guard = self.banlist.lock().unwrap(); - guard.insert(address.clone(), now); + guard[shard].insert(address.clone(), now); } /// Clear the replica to receive traffic again. Takes effect immediately /// for all new transactions. - pub fn unban(&self, address: &Address) { + pub fn unban(&self, address: &Address, shard: usize) { let mut guard = self.banlist.lock().unwrap(); - guard.remove(address); + guard[shard].remove(address); } /// Check if a replica can serve traffic. If all replicas are banned, /// we unban all of them. Better to try then not to. - pub fn is_banned(&self, address: &Address) -> bool { + pub fn is_banned(&self, address: &Address, shard: usize) -> bool { let mut guard = self.banlist.lock().unwrap(); // Everything is banned, nothig is banned - if guard.len() == self.databases.len() { + if guard[shard].len() == self.databases[shard].len() { guard.clear(); drop(guard); println!(">> Unbanning all replicas."); @@ -158,12 +164,12 @@ impl ConnectionPool { } // I expect this to miss 99.9999% of the time. - match guard.get(address) { + match guard[shard].get(address) { Some(timestamp) => { let now = chrono::offset::Utc::now().naive_utc(); if now.timestamp() - timestamp.timestamp() > BAN_TIME { // 1 minute - guard.remove(address); + guard[shard].remove(address); false } else { true From e4ed5ee0794c193b3197a7a4476bea9285bdc2e6 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 6 Feb 2022 11:22:09 -0800 Subject: [PATCH 017/551] rename --- .gitignore | 1 + Cargo.lock | 28 ++++++++++++++-------------- Cargo.toml | 2 +- tests/python/tests.py | 6 +++--- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index ea8c4bf7f..50ccb252b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +*.deb diff --git a/Cargo.lock b/Cargo.lock index 8cf1dcbd4..00053f9c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -305,6 +305,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "pgcat" +version = "0.1.0" +dependencies = [ + "async-trait", + "bb8", + "bytes", + "chrono", + "md-5", + "rand", + "sha-1", + "tokio", +] + [[package]] name = "pin-project-lite" version = "0.2.8" @@ -341,20 +355,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rabbit" -version = "0.1.0" -dependencies = [ - "async-trait", - "bb8", - "bytes", - "chrono", - "md-5", - "rand", - "sha-1", - "tokio", -] - [[package]] name = "rand" version = "0.8.4" diff --git a/Cargo.toml b/Cargo.toml index bc8034a05..d29a9c27f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "rabbit" +name = "pgcat" version = "0.1.0" edition = "2021" diff --git a/tests/python/tests.py b/tests/python/tests.py index 34f19336e..8eb47f6a4 100644 --- a/tests/python/tests.py +++ b/tests/python/tests.py @@ -1,11 +1,11 @@ import psycopg2 -conn = psycopg2.connect("postgres://random:password@127.0.0.1:5433/db") +conn = psycopg2.connect("postgres://random:password@127.0.0.1:6432/db") cur = conn.cursor() -cur.execute("SELECT $1", [1234]); +cur.execute("SELECT 1"); res = cur.fetchall() print(res) -conn.commit() \ No newline at end of file +# conn.commit() \ No newline at end of file From 822757007850170fd1bd280ee00c19ef0d08cee3 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 6 Feb 2022 12:52:59 -0800 Subject: [PATCH 018/551] nuance --- src/pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pool.rs b/src/pool.rs index 5515267af..2a54d552a 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -157,7 +157,7 @@ impl ConnectionPool { // Everything is banned, nothig is banned if guard[shard].len() == self.databases[shard].len() { - guard.clear(); + guard[shard].clear(); drop(guard); println!(">> Unbanning all replicas."); return false; From 7e0d1a727f771853223d280d85a7360f4be3b53b Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 7 Feb 2022 11:15:33 -0800 Subject: [PATCH 019/551] more user friendly server error msg --- src/server.rs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/server.rs b/src/server.rs index cfce83727..96c619c73 100644 --- a/src/server.rs +++ b/src/server.rs @@ -86,10 +86,8 @@ impl Server { md5_password(&mut stream, user, password, &salt[..]).await?; } - // We're in! - 0 => { - println!(">> Server authentication successful!"); - } + // Authentication handshake complete. + 0 => (), _ => { println!(">> Unsupported authentication mechanism: {}", code); @@ -99,7 +97,23 @@ impl Server { } 'E' => { - println!(">> Database error"); + let error_code = match stream.read_u8().await { + Ok(error_code) => error_code, + Err(_) => return Err(Error::SocketError), + }; + + match error_code { + 0 => (), // Terminator + _ => { + let mut error = vec![0u8; len as usize - 4 - 1]; + match stream.read_exact(&mut error).await { + Ok(_) => (), + Err(_) => return Err(Error::SocketError), + }; + + println!(">> Server error: {}", String::from_utf8_lossy(&error)); + } + }; return Err(Error::ServerError); } From f6e9827dbfcb17af45fea65af23850dbe1a0c8a6 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 09:25:59 -0800 Subject: [PATCH 020/551] config support; started more sharding --- Cargo.lock | 29 ++++++++++++++++ Cargo.toml | 5 ++- pgcat.toml | 60 +++++++++++++++++++++++++++++++++ src/client.rs | 5 +-- src/config.rs | 86 +++++++++++++++++++++++++++++++++++++++-------- src/errors.rs | 1 + src/main.rs | 50 +++++++++++++++------------- src/pool.rs | 68 ++++++++++++++++++++++++++++++++++++-- src/server.rs | 22 +++++++++++++ src/sharding.rs | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 373 insertions(+), 41 deletions(-) create mode 100644 pgcat.toml diff --git a/Cargo.lock b/Cargo.lock index 00053f9c2..f4909f0d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -315,8 +315,11 @@ dependencies = [ "chrono", "md-5", "rand", + "serde", + "serde_derive", "sha-1", "tokio", + "toml", ] [[package]] @@ -410,6 +413,23 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "serde" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" + +[[package]] +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha-1" version = "0.10.0" @@ -494,6 +514,15 @@ dependencies = [ "syn", ] +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + [[package]] name = "typenum" version = "1.15.0" diff --git a/Cargo.toml b/Cargo.toml index d29a9c27f..3cb0a6e86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,4 +13,7 @@ bb8 = "*" async-trait = "*" rand = "*" chrono = "0.4" -sha-1 = "*" \ No newline at end of file +sha-1 = "*" +toml = "*" +serde = "*" +serde_derive = "*" \ No newline at end of file diff --git a/pgcat.toml b/pgcat.toml new file mode 100644 index 000000000..bb410495b --- /dev/null +++ b/pgcat.toml @@ -0,0 +1,60 @@ +# +# PgCat config example. +# + +# +# General pooler settings +[general] + +# What IP to run on, 0.0.0.0 means accessible from everywhere. +host = "0.0.0.0" + +# Port to run on, same as PgBouncer used in this example. +port = 6432 + +# How many connections to allocate per server. +pool_size = 15 + +# Pool mode (see PgBouncer docs for more). +# session: one server connection per connected client +# transaction: one server connection per client transaction +pool_mode = "transaction" + +# How long to wait before aborting a server connection (ms). +connect_timeout = 5000 + +# How much time to give `SELECT 1` health check query to return with a result (ms). +healthcheck_timeout = 1000 + +# For how long to ban a server if it fails a health check (seconds). +ban_time = 60 # Seconds + +# +# User to use for authentication against the server. +[user] +name = "lev" +password = "lev" + + +# +# Shards in the cluster +[shards] + +# Shard 0 +[shards.0] + +# [ host, port ] +servers = [ + [ "127.0.0.1", 5432 ], + [ "localhost", 5432 ], +] +# Database name (e.g. "postgres") +database = "lev" + +[shards.1] +# [ host, port ] +servers = [ + [ "127.0.0.1", 5432 ], + [ "localhost", 5432 ], +] +database = "lev" \ No newline at end of file diff --git a/src/client.rs b/src/client.rs index 5502a3e02..b7e4b153c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -48,6 +48,7 @@ impl Client { pub async fn startup( mut stream: TcpStream, client_server_map: ClientServerMap, + transaction_mode: bool, ) -> Result { loop { // Could be StartupMessage or SSLRequest @@ -100,7 +101,7 @@ impl Client { write: write, buffer: BytesMut::with_capacity(8196), cancel_mode: false, - transaction_mode: true, + transaction_mode: transaction_mode, process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, @@ -119,7 +120,7 @@ impl Client { write: write, buffer: BytesMut::with_capacity(8196), cancel_mode: true, - transaction_mode: true, + transaction_mode: transaction_mode, process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, diff --git a/src/config.rs b/src/config.rs index 736d6a5b7..bfc5619ad 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,24 +1,84 @@ +use serde_derive::Deserialize; +use std::collections::HashMap; +use std::path::Path; +use tokio::fs::File; +use tokio::io::AsyncReadExt; +use toml; + +use crate::errors::Error; + #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)] pub struct Address { pub host: String, pub port: String, } -#[derive(Clone, PartialEq, Hash, std::cmp::Eq)] +#[derive(Clone, PartialEq, Hash, std::cmp::Eq, Deserialize, Debug)] pub struct User { pub name: String, pub password: String, } -// #[derive(Clone)] -// pub struct Config { -// pools: HashMap>, -// } - -// impl Config { -// pub fn new() -> Config { -// Config { -// pools: HashMap::new(), -// } -// } -// } +#[derive(Deserialize, Debug, Clone)] +pub struct General { + pub host: String, + pub port: i16, + pub pool_size: u32, + pub pool_mode: String, + pub connect_timeout: u64, + pub healthcheck_timeout: u64, + pub ban_time: i64, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Shard { + pub servers: Vec<(String, u16)>, + pub database: String, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct Config { + pub general: General, + pub user: User, + pub shards: HashMap, +} + +pub async fn parse(path: &str) -> Result { + // let path = Path::new(path); + let mut contents = String::new(); + let mut file = match File::open(path).await { + Ok(file) => file, + Err(err) => { + println!("> Config error: {:?}", err); + return Err(Error::BadConfig); + } + }; + + match file.read_to_string(&mut contents).await { + Ok(_) => (), + Err(err) => { + println!("> Config error: {:?}", err); + return Err(Error::BadConfig); + } + }; + + // let config: toml::Value = match toml::from_str(&contents) { + // Ok(config) => config, + // Err(err) => { + // println!("> Config error: {:?}", err); + // return Err(Error::BadConfig); + // } + // }; + + // println!("Config: {:?}", config); + + let config: Config = match toml::from_str(&contents) { + Ok(config) => config, + Err(err) => { + println!("> Config error: {:?}", err); + return Err(Error::BadConfig); + } + }; + + Ok(config) +} diff --git a/src/errors.rs b/src/errors.rs index cb7e756ab..5ee43e3f0 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -7,4 +7,5 @@ pub enum Error { ServerError, ServerTimeout, DirtyServer, + BadConfig, } diff --git a/src/main.rs b/src/main.rs index dcab2b28b..63972bfa1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,10 @@ extern crate async_trait; extern crate bb8; extern crate bytes; extern crate md5; +extern crate serde; +extern crate serde_derive; extern crate tokio; +extern crate toml; use tokio::net::TcpListener; @@ -42,8 +45,15 @@ use pool::{ClientServerMap, ConnectionPool}; async fn main() { println!("> Welcome to PgCat! Meow."); - let addr = "0.0.0.0:6432"; - let listener = match TcpListener::bind(addr).await { + let config = match config::parse("pgcat.toml").await { + Ok(config) => config, + Err(err) => { + return; + } + }; + + let addr = format!("{}:{}", config.general.host, config.general.port); + let listener = match TcpListener::bind(&addr).await { Ok(sock) => sock, Err(err) => { println!("> Error: {:?}", err); @@ -53,28 +63,21 @@ async fn main() { println!("> Running on {}", addr); + // Tracks which client is connected to which server for query cancellation. let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); - // Replica pool. - let addresses = vec![ - Address { - host: "127.0.0.1".to_string(), - port: "5432".to_string(), - }, - Address { - host: "localhost".to_string(), - port: "5433".to_string(), - }, - ]; - - let user = User { - name: "lev".to_string(), - password: "lev".to_string(), - }; + println!("> Pool size: {}", config.general.pool_size); + println!("> Pool mode: {}", config.general.pool_mode); + println!("> Ban time: {}s", config.general.ban_time); + println!( + "> Healthcheck timeout: {}ms", + config.general.healthcheck_timeout + ); - let database = "lev"; + let pool = ConnectionPool::from_config(config.clone(), client_server_map.clone()).await; + let transaction_mode = config.general.pool_mode == "transaction"; - let pool = ConnectionPool::new(addresses, user, database, client_server_map.clone()).await; + println!("> Waiting for clients..."); loop { let pool = pool.clone(); @@ -90,9 +93,12 @@ async fn main() { // Client goes to another thread, bye. tokio::task::spawn(async move { - println!(">> Client {:?} connected.", addr); + println!( + ">> Client {:?} connected, transaction pooling: {}", + addr, transaction_mode + ); - match client::Client::startup(socket, client_server_map).await { + match client::Client::startup(socket, client_server_map, transaction_mode).await { Ok(mut client) => { println!(">> Client {:?} authenticated successfully!", addr); diff --git a/src/pool.rs b/src/pool.rs index 2a54d552a..073c20a54 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection}; use chrono::naive::NaiveDateTime; -use crate::config::{Address, User}; +use crate::config::{Address, Config, User}; use crate::errors::Error; use crate::server::Server; @@ -31,15 +31,17 @@ const CONNECT_TIMEOUT: u64 = 5000; // How much time to give the server to answer a SELECT 1 query. const HEALTHCHECK_TIMEOUT: u64 = 1000; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct ConnectionPool { databases: Vec>>, addresses: Vec>, round_robin: Counter, banlist: BanList, + healthcheck_timeout: u64, } impl ConnectionPool { + // Construct the connection pool for a single-shard cluster. pub async fn new( addresses: Vec
, user: User, @@ -71,10 +73,70 @@ impl ConnectionPool { addresses: vec![addresses], round_robin: Arc::new(AtomicUsize::new(0)), banlist: Arc::new(Mutex::new(vec![HashMap::new()])), + healthcheck_timeout: HEALTHCHECK_TIMEOUT, } } - /// Get a connection from the pool. Either round-robin or pick a specific one in case they are sharded. + /// Construct the connection pool from a config file. + pub async fn from_config(config: Config, client_server_map: ClientServerMap) -> ConnectionPool { + let mut shards = Vec::new(); + let mut addresses = Vec::new(); + let mut banlist = Vec::new(); + let mut shard_ids = config + .shards + .clone() + .into_keys() + .map(|x| x.to_string()) + .collect::>(); + shard_ids.sort_by_key(|k| k.parse::().unwrap()); + + for shard in shard_ids { + let shard = &config.shards[&shard]; + let mut pools = Vec::new(); + let mut replica_addresses = Vec::new(); + + for server in &shard.servers { + let address = Address { + host: server.0.clone(), + port: server.1.to_string(), + }; + + let manager = ServerPool::new( + address.clone(), + config.user.clone(), + &shard.database, + client_server_map.clone(), + ); + + let pool = Pool::builder() + .max_size(config.general.pool_size) + .connection_timeout(std::time::Duration::from_millis( + config.general.connect_timeout, + )) + .test_on_check_out(false) + .build(manager) + .await + .unwrap(); + + pools.push(pool); + replica_addresses.push(address); + } + + shards.push(pools); + addresses.push(replica_addresses); + banlist.push(HashMap::new()); + } + + ConnectionPool { + databases: shards, + addresses: addresses, + round_robin: Arc::new(AtomicUsize::new(0)), + banlist: Arc::new(Mutex::new(banlist)), + healthcheck_timeout: config.general.healthcheck_timeout, + } + } + + /// Get a connection from the pool. pub async fn get( &self, shard: Option, diff --git a/src/server.rs b/src/server.rs index 96c619c73..0a7e31fc6 100644 --- a/src/server.rs +++ b/src/server.rs @@ -15,17 +15,38 @@ use crate::ClientServerMap; /// Server state. pub struct Server { + // Server host, e.g. localhost host: String, + + // Server port: e.g. 5432 port: String, + + // Buffered read socket read: BufReader, + + // Unbuffered write socket (our client code buffers) write: OwnedWriteHalf, + + // Our server response buffer buffer: BytesMut, + + // Server information the server sent us over on startup server_info: BytesMut, + + // Backend id and secret key used for query cancellation. backend_id: i32, secret_key: i32, + + // Is the server inside a transaction at the moment. in_transaction: bool, + + // Is there more data for the client to read. data_available: bool, + + // Is the server broken? We'll remote it from the pool if so. bad: bool, + + // Mapping of clients and servers used for query cancellation. client_server_map: ClientServerMap, } @@ -48,6 +69,7 @@ impl Server { } }; + // Send the startup packet. startup(&mut stream, user, database).await?; let mut server_info = BytesMut::with_capacity(25); diff --git a/src/sharding.rs b/src/sharding.rs index bf976e50f..39c955fe0 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -1,5 +1,8 @@ use sha1::{Digest, Sha1}; +// https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/include/catalog/partition.h#L20 +const PARTITION_HASH_SEED: u64 = 0x7A5B22367996DCFD; + pub struct Sharder { shards: usize, } @@ -9,6 +12,8 @@ impl Sharder { Sharder { shards: shards } } + /// Use SHA1 to pick a shard for the key. The key can be anything, + /// including an int or a string. pub fn sha1(&self, key: &[u8]) -> usize { let mut hasher = Sha1::new(); hasher.update(key); @@ -17,6 +22,81 @@ impl Sharder { let i = u32::from_le_bytes(result[result.len() - 4..result.len()].try_into().unwrap()); i as usize % self.shards } + + /// Hash function used by Postgres to determine which partition + /// to put the row in when using HASH(column) partitioning. + /// Source: https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/common/hashfn.c#L631 + pub fn pg_bigint_hash(&self, key: i64) -> usize { + let mut lohalf = key as u32; + let hihalf = (key >> 32) as u32; + println!("{}, {}", lohalf, hihalf); + lohalf ^= if key >= 0 { hihalf } else { !hihalf }; + println!("Low half: {}", lohalf); + Self::pg_u32_hash(lohalf) as usize % self.shards + } + + fn rot(x: u32, k: u32) -> u32 { + ((x) << (k)) | ((x) >> (32 - (k))) + } + + #[inline] + fn mix(mut a: u32, mut b: u32, mut c: u32) -> (u32, u32, u32) { + a = a.wrapping_sub(c); + a ^= Self::rot(c, 4); + c = c.wrapping_add(b); + b = b.wrapping_add(a); + b ^= Self::rot(a, 6); + a = a.wrapping_add(c); + c = c.wrapping_add(b); + c ^= Self::rot(b, 8); + b = b.wrapping_add(a); + a = a.wrapping_add(c); + a ^= Self::rot(c, 16); + c = c.wrapping_add(b); + b = b.wrapping_add(a); + b ^= Self::rot(a, 19); + a = a.wrapping_add(c); + c = c.wrapping_add(b); + c ^= Self::rot(b, 4); + b = b.wrapping_add(a); + (a, b, c) + } + + #[inline] + fn _final(mut a: u32, mut b: u32, mut c: u32) -> (u32, u32, u32) { + c ^= b; + c = c.wrapping_add(Self::rot(b, 14)); + a ^= c; + a = a.wrapping_add(Self::rot(c, 11)); + b ^= a; + b = b.wrapping_add(Self::rot(a, 25)); + c ^= b; + c = c.wrapping_add(Self::rot(b, 16)); + a ^= c; + a = a.wrapping_add(Self::rot(c, 4)); + b ^= a; + b = b.wrapping_add(Self::rot(a, 14)); + c ^= b; + c = c.wrapping_add(Self::rot(b, 24)); + (a, b, c) + } + + fn pg_u32_hash(val: u32) -> u64 { + let mut a: u32 = 0x9e3779b9 + 4 + 3923095; + let mut b = a; + let c = a; + let seed = PARTITION_HASH_SEED; + + a = a.wrapping_add((seed >> 32) as u32); + b = b.wrapping_add(seed as u32); + let (mut a, b, c) = Self::mix(a, b, c); + + a = a.wrapping_add(val); + + let (a, b, c) = Self::_final(a, b, c); + + (b as u64) << 32 | c as u64 + } } #[cfg(test)] @@ -30,4 +110,12 @@ mod test { let shard = sharder.sha1(key); assert_eq!(shard, 1); } + + #[test] + fn test_pg_bigint_hash() { + let sharder = Sharder::new(4); + let key = 23423423 as i64; + let shard = sharder.pg_bigint_hash(key); + assert_eq!(shard, 0); + } } From 100f9550d5647141e9b2a11cc56769ffd761dd51 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 09:28:53 -0800 Subject: [PATCH 021/551] ban time --- src/pool.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pool.rs b/src/pool.rs index 073c20a54..874fd615f 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -38,6 +38,7 @@ pub struct ConnectionPool { round_robin: Counter, banlist: BanList, healthcheck_timeout: u64, + ban_time: i64, } impl ConnectionPool { @@ -74,6 +75,7 @@ impl ConnectionPool { round_robin: Arc::new(AtomicUsize::new(0)), banlist: Arc::new(Mutex::new(vec![HashMap::new()])), healthcheck_timeout: HEALTHCHECK_TIMEOUT, + ban_time: BAN_TIME, } } @@ -133,6 +135,7 @@ impl ConnectionPool { round_robin: Arc::new(AtomicUsize::new(0)), banlist: Arc::new(Mutex::new(banlist)), healthcheck_timeout: config.general.healthcheck_timeout, + ban_time: config.general.ban_time, } } @@ -229,7 +232,7 @@ impl ConnectionPool { match guard[shard].get(address) { Some(timestamp) => { let now = chrono::offset::Utc::now().naive_utc(); - if now.timestamp() - timestamp.timestamp() > BAN_TIME { + if now.timestamp() - timestamp.timestamp() > self.ban_time { // 1 minute guard[shard].remove(address); false From 81af5a87e6d5caeaa60430c7f3af49f46b9fdf99 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 09:33:20 -0800 Subject: [PATCH 022/551] comments --- src/messages.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/messages.rs b/src/messages.rs index 9a74c7f70..58459f97e 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -13,6 +13,7 @@ use crate::errors::Error; // const SERVER_VESION: &str = "12.9 (Ubuntu 12.9-0ubuntu0.20.04.1)"; +/// Tell the client that authentication handshake completed successfully. pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { let mut auth_ok = BytesMut::with_capacity(9); @@ -23,6 +24,8 @@ pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { Ok(write_all(stream, auth_ok).await?) } +/// Send server parameters to the client. This will tell the client +/// what server version and what's the encoding we're using. pub async fn server_parameters(stream: &mut TcpStream) -> Result<(), Error> { let client_encoding = BytesMut::from(&b"client_encoding\0UTF8\0"[..]); let server_version = @@ -44,6 +47,8 @@ pub async fn server_parameters(stream: &mut TcpStream) -> Result<(), Error> { Ok(write_all(stream, res).await?) } +/// Give the client the process_id and secret we generated +/// used in query cancellation. pub async fn backend_key_data( stream: &mut TcpStream, backend_id: i32, @@ -57,6 +62,7 @@ pub async fn backend_key_data( Ok(write_all(stream, key_data).await?) } +/// Tell the client we're ready for another query. pub async fn ready_for_query(stream: &mut TcpStream) -> Result<(), Error> { let mut bytes = BytesMut::with_capacity(5); @@ -67,6 +73,8 @@ pub async fn ready_for_query(stream: &mut TcpStream) -> Result<(), Error> { Ok(write_all(stream, bytes).await?) } +/// Send the startup packet the server. We're pretending we're a Pg client. +/// This tells the server which user we are and what database we want. pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Result<(), Error> { let mut bytes = BytesMut::with_capacity(25); @@ -96,6 +104,8 @@ pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Resu } } +/// Send password challenge response to the server. +/// This is the MD5 challenge. pub async fn md5_password( stream: &mut TcpStream, user: &str, @@ -128,6 +138,7 @@ pub async fn md5_password( Ok(write_all(stream, message).await?) } +/// Write all data in the buffer to the TcpStream. pub async fn write_all(stream: &mut TcpStream, buf: BytesMut) -> Result<(), Error> { match stream.write_all(&buf).await { Ok(_) => Ok(()), @@ -135,6 +146,7 @@ pub async fn write_all(stream: &mut TcpStream, buf: BytesMut) -> Result<(), Erro } } +/// Write all the data in the buffer to the TcpStream, write owned half (see mpsc). pub async fn write_all_half(stream: &mut OwnedWriteHalf, buf: BytesMut) -> Result<(), Error> { match stream.write_all(&buf).await { Ok(_) => Ok(()), From 4642c5193bdcc65b762e293cd8f6d87c85ebf6bd Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 10:05:10 -0800 Subject: [PATCH 023/551] fix some bugs in sharding - still doesnt work --- src/sharding.rs | 52 ++++++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/src/sharding.rs b/src/sharding.rs index 39c955fe0..e9cd3650b 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -29,9 +29,7 @@ impl Sharder { pub fn pg_bigint_hash(&self, key: i64) -> usize { let mut lohalf = key as u32; let hihalf = (key >> 32) as u32; - println!("{}, {}", lohalf, hihalf); lohalf ^= if key >= 0 { hihalf } else { !hihalf }; - println!("Low half: {}", lohalf); Self::pg_u32_hash(lohalf) as usize % self.shards } @@ -44,45 +42,51 @@ impl Sharder { a = a.wrapping_sub(c); a ^= Self::rot(c, 4); c = c.wrapping_add(b); - b = b.wrapping_add(a); + + b = b.wrapping_sub(a); b ^= Self::rot(a, 6); a = a.wrapping_add(c); - c = c.wrapping_add(b); + + c = c.wrapping_sub(b); c ^= Self::rot(b, 8); b = b.wrapping_add(a); - a = a.wrapping_add(c); + + a = a.wrapping_sub(c); a ^= Self::rot(c, 16); c = c.wrapping_add(b); - b = b.wrapping_add(a); + + b = b.wrapping_sub(a); b ^= Self::rot(a, 19); a = a.wrapping_add(c); - c = c.wrapping_add(b); + + c = c.wrapping_sub(b); c ^= Self::rot(b, 4); b = b.wrapping_add(a); + (a, b, c) } #[inline] fn _final(mut a: u32, mut b: u32, mut c: u32) -> (u32, u32, u32) { c ^= b; - c = c.wrapping_add(Self::rot(b, 14)); + c = c.wrapping_sub(Self::rot(b, 14)); a ^= c; - a = a.wrapping_add(Self::rot(c, 11)); + a = a.wrapping_sub(Self::rot(c, 11)); b ^= a; - b = b.wrapping_add(Self::rot(a, 25)); + b = b.wrapping_sub(Self::rot(a, 25)); c ^= b; - c = c.wrapping_add(Self::rot(b, 16)); + c = c.wrapping_sub(Self::rot(b, 16)); a ^= c; - a = a.wrapping_add(Self::rot(c, 4)); + a = a.wrapping_sub(Self::rot(c, 4)); b ^= a; - b = b.wrapping_add(Self::rot(a, 14)); + b = b.wrapping_sub(Self::rot(a, 14)); c ^= b; - c = c.wrapping_add(Self::rot(b, 24)); + c = c.wrapping_sub(Self::rot(b, 24)); (a, b, c) } - fn pg_u32_hash(val: u32) -> u64 { - let mut a: u32 = 0x9e3779b9 + 4 + 3923095; + fn pg_u32_hash(k: u32) -> u64 { + let mut a: u32 = 0x9e3779b9 as u32 + std::mem::size_of::() as u32 + 3923095 as u32; let mut b = a; let c = a; let seed = PARTITION_HASH_SEED; @@ -91,11 +95,11 @@ impl Sharder { b = b.wrapping_add(seed as u32); let (mut a, b, c) = Self::mix(a, b, c); - a = a.wrapping_add(val); + a = a.wrapping_add(k); let (a, b, c) = Self::_final(a, b, c); - (b as u64) << 32 | c as u64 + ((b as u64) << 32) | (c as u64) } } @@ -113,9 +117,17 @@ mod test { #[test] fn test_pg_bigint_hash() { - let sharder = Sharder::new(4); - let key = 23423423 as i64; + let sharder = Sharder::new(2); + let key = 1 as i64; let shard = sharder.pg_bigint_hash(key); assert_eq!(shard, 0); + + let key = 2 as i64; + let shard = sharder.pg_bigint_hash(key); + assert_eq!(shard, 0); + + let key = 3 as i64; + let shard = sharder.pg_bigint_hash(key); + assert_eq!(shard, 1); } } From d38833c26411148a310f3887c6ce2c67d7764ffc Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 11:10:28 -0800 Subject: [PATCH 024/551] pg sharding works --- src/sharding.rs | 63 +++++++++++++++++++++++++++++----------- tests/sharding/setup.sql | 26 +++++++++++++++++ 2 files changed, 72 insertions(+), 17 deletions(-) create mode 100644 tests/sharding/setup.sql diff --git a/src/sharding.rs b/src/sharding.rs index e9cd3650b..57cf8075b 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -26,15 +26,17 @@ impl Sharder { /// Hash function used by Postgres to determine which partition /// to put the row in when using HASH(column) partitioning. /// Source: https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/common/hashfn.c#L631 - pub fn pg_bigint_hash(&self, key: i64) -> usize { + /// Supports only 1 bigint at the moment, but we can add more later. + pub fn pg_bigint_hash(&self, key: i64) -> u64 { let mut lohalf = key as u32; let hihalf = (key >> 32) as u32; lohalf ^= if key >= 0 { hihalf } else { !hihalf }; - Self::pg_u32_hash(lohalf) as usize % self.shards + Self::combine(0, Self::pg_u32_hash(lohalf)) % self.shards as u64 } + #[inline] fn rot(x: u32, k: u32) -> u32 { - ((x) << (k)) | ((x) >> (32 - (k))) + (x << k) | (x >> (32 - k)) } #[inline] @@ -85,19 +87,27 @@ impl Sharder { (a, b, c) } + #[inline] + fn combine(mut a: u64, b: u64) -> u64 { + a ^= b + .wrapping_add(0x49a0f4dd15e5a8e3 as u64) + .wrapping_add(a << 54) + .wrapping_add(a >> 7); + a + } + fn pg_u32_hash(k: u32) -> u64 { let mut a: u32 = 0x9e3779b9 as u32 + std::mem::size_of::() as u32 + 3923095 as u32; let mut b = a; let c = a; - let seed = PARTITION_HASH_SEED; - a = a.wrapping_add((seed >> 32) as u32); - b = b.wrapping_add(seed as u32); + a = a.wrapping_add((PARTITION_HASH_SEED >> 32) as u32); + b = b.wrapping_add(PARTITION_HASH_SEED as u32); let (mut a, b, c) = Self::mix(a, b, c); a = a.wrapping_add(k); - let (a, b, c) = Self::_final(a, b, c); + let (_a, b, c) = Self::_final(a, b, c); ((b as u64) << 32) | (c as u64) } @@ -117,17 +127,36 @@ mod test { #[test] fn test_pg_bigint_hash() { - let sharder = Sharder::new(2); - let key = 1 as i64; - let shard = sharder.pg_bigint_hash(key); - assert_eq!(shard, 0); + let sharder = Sharder::new(5); - let key = 2 as i64; - let shard = sharder.pg_bigint_hash(key); - assert_eq!(shard, 0); + let shard_0 = vec![1, 4, 5, 14, 19, 39, 40, 46, 47, 53]; - let key = 3 as i64; - let shard = sharder.pg_bigint_hash(key); - assert_eq!(shard, 1); + for v in shard_0 { + assert_eq!(sharder.pg_bigint_hash(v), 0); + } + + let shard_1 = vec![2, 3, 11, 17, 21, 23, 30, 49, 51, 54]; + + for v in shard_1 { + assert_eq!(sharder.pg_bigint_hash(v), 1); + } + + let shard_2 = vec![6, 7, 15, 16, 18, 20, 25, 28, 34, 35]; + + for v in shard_2 { + assert_eq!(sharder.pg_bigint_hash(v), 2); + } + + let shard_3 = vec![8, 12, 13, 22, 29, 31, 33, 36, 41, 43]; + + for v in shard_3 { + assert_eq!(sharder.pg_bigint_hash(v), 3); + } + + let shard_4 = vec![9, 10, 24, 26, 27, 32, 37, 38, 42, 45]; + + for v in shard_4 { + assert_eq!(sharder.pg_bigint_hash(v), 4); + } } } diff --git a/tests/sharding/setup.sql b/tests/sharding/setup.sql new file mode 100644 index 000000000..e802ead23 --- /dev/null +++ b/tests/sharding/setup.sql @@ -0,0 +1,26 @@ +DROP TABLE IF EXISTS shards CASCADE; + +CREATE TABLE shards ( + id BIGINT, + value VARCHAR +) PARTITION BY HASH (id); + +-- DROP TABLE IF EXISTS shard_0; +CREATE TABLE shard_0 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 0); +-- DROP TABLE IF EXISTS shard_1; +CREATE TABLE shard_1 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 1); +-- DROP TABLE IF EXISTS shard_2; +CREATE TABLE shard_2 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 2); +-- DROP TABLE IF EXISTS shard_3; +CREATE TABLE shard_3 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 3); +-- DROP TABLE IF EXISTS shard_4; +CREATE TABLE shard_4 PARTITION OF shards FOR VALUES WITH (MODULUS 5, REMAINDER 4); + + +INSERT INTO shards SELECT generate_series(1, 500), 'value'; + +SELECT * FROM shard_0 ORDER BY id LIMIT 10; +SELECT * FROM shard_1 ORDER BY id LIMIT 10; +SELECT * FROM shard_2 ORDER BY id LIMIT 10; +SELECT * FROM shard_3 ORDER BY id LIMIT 10; +SELECT * FROM shard_4 ORDER BY id LIMIT 10; \ No newline at end of file From 0df727f72c489ff29e00490d0f4f7a0358e9db20 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 11:14:20 -0800 Subject: [PATCH 025/551] usize --- src/sharding.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sharding.rs b/src/sharding.rs index 57cf8075b..b72923491 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -27,11 +27,11 @@ impl Sharder { /// to put the row in when using HASH(column) partitioning. /// Source: https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/common/hashfn.c#L631 /// Supports only 1 bigint at the moment, but we can add more later. - pub fn pg_bigint_hash(&self, key: i64) -> u64 { + pub fn pg_bigint_hash(&self, key: i64) -> usize { let mut lohalf = key as u32; let hihalf = (key >> 32) as u32; lohalf ^= if key >= 0 { hihalf } else { !hihalf }; - Self::combine(0, Self::pg_u32_hash(lohalf)) % self.shards as u64 + Self::combine(0, Self::pg_u32_hash(lohalf)) as usize % self.shards as usize } #[inline] @@ -125,6 +125,9 @@ mod test { assert_eq!(shard, 1); } + // See tests/sharding/setup.sql + // The output of those SELECT statements will match this test, + // confirming that we implemented Postgres BIGINT hashing correctly. #[test] fn test_pg_bigint_hash() { let sharder = Sharder::new(5); From 83b7740248d30791b228a76b0710bf86e96d6fc9 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 11:21:19 -0800 Subject: [PATCH 026/551] readme --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 577c25427..d7217e9c8 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Okay, this is just basic stuff, but we support cancelling queries. If you know t this might be relevant given than this is a transactional pooler but if you're new to Pg, don't worry about it, it works. ### Round-robin load balancing -This is the novel part. PgBouncer doesn't support it and suggests we use DNS of a TCP proxy instead. +This is the novel part. PgBouncer doesn't support it and suggests we use DNS or a TCP proxy instead. We prefer to have everything as part of one package; arguably, it's easier to understand and optimize. This pooler will round-robin between multiple replicas keeping load reasonably even. @@ -64,10 +64,16 @@ failing over could bring even more load and tip over the remaining healthy-ish r either lose 1/x of your traffic or risk losing it all eventually. Ideally you overprovision your system, so you don't necessarily need to make this choice :-). +### Sharding +We're implemeting Postgres' `PARTITION BY HASH` sharding function for `BIGINT` fields. This works well for tables that use `BIGSERIAL` primary key which I think is common enough these days. We can also add many more functions here, but this is a good start. See `src/sharding.rs` and `tests/sharding/setup.sql` for more details on the implementation. + +The biggest advantage of using this sharding function is that anyone can shard the dataset using Postgres partitions +while also access it for both reads and writes using this pooler. No custom obscure sharding function is needed and database sharding can be done entirely in Postgres. + ## Missing -1. Sharding. Soon :-). +1. Query routing based on shard. 2. Authentication, ehem, this proxy is letting anyone in at the moment. ## Benchmarks From 61b2422e83c58781f220911499f18db1b3dee781 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 13:11:50 -0800 Subject: [PATCH 027/551] sharded query routing --- Cargo.lock | 27 ++++++++++++++++++++++ Cargo.toml | 3 ++- src/client.rs | 60 ++++++++++++++++++++++++++++++++++++++++++++++--- src/messages.rs | 17 ++++++++++++++ 4 files changed, 103 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4909f0d7..0e6f79e17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + [[package]] name = "async-trait" version = "0.1.52" @@ -315,6 +324,7 @@ dependencies = [ "chrono", "md-5", "rand", + "regex", "serde", "serde_derive", "sha-1", @@ -407,6 +417,23 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + [[package]] name = "scopeguard" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 3cb0a6e86..019c15391 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,4 +16,5 @@ chrono = "0.4" sha-1 = "*" toml = "*" serde = "*" -serde_derive = "*" \ No newline at end of file +serde_derive = "*" +regex = "1" \ No newline at end of file diff --git a/src/client.rs b/src/client.rs index b7e4b153c..0ba97d0c0 100644 --- a/src/client.rs +++ b/src/client.rs @@ -3,6 +3,7 @@ /// and this module implements that. use bytes::{Buf, BufMut, BytesMut}; use rand::{distributions::Alphanumeric, Rng}; +use regex::Regex; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::net::TcpStream; @@ -11,6 +12,9 @@ use crate::errors::Error; use crate::messages::*; use crate::pool::{ClientServerMap, ConnectionPool}; use crate::server::Server; +use crate::sharding::Sharder; + +const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+';"; /// The client state. One of these is created per client. pub struct Client { @@ -39,6 +43,9 @@ pub struct Client { // Clients are mapped to servers while they use them. This allows a client // to connect and cancel a query. client_server_map: ClientServerMap, + + // sharding regex + sharding_regex: Regex, } impl Client { @@ -50,6 +57,8 @@ impl Client { client_server_map: ClientServerMap, transaction_mode: bool, ) -> Result { + let sharding_regex = Regex::new(SHARDING_REGEX).unwrap(); + loop { // Could be StartupMessage or SSLRequest // which makes this variable length. @@ -105,6 +114,7 @@ impl Client { process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, + sharding_regex: sharding_regex, }); } @@ -124,6 +134,7 @@ impl Client { process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, + sharding_regex: sharding_regex, }); } @@ -156,6 +167,12 @@ impl Client { return Ok(Server::cancel(&address, &port, process_id, secret_key).await?); } + // Active shard we're talking to. + // The lifetime of this depends on the pool mode: + // - if in session mode, this lives until client disconnects or changes it, + // - if in transaction mode, this lives for the duration of one transaction. + let mut shard: Option = None; + loop { // Read a complete message from the client, which normally would be // either a `Q` (query) or `P` (prepare, extended protocol). @@ -164,15 +181,23 @@ impl Client { // SET sharding_context.key = '1234'; let mut message = read_message(&mut self.read).await?; - // TODO: parse the message here. If it's part of our protocol, - // don't grab a server yet and continue loop. + // Parse for special select shard command. + // SET SHARDING KEY TO 'bigint'; + match self.select_shard(message.clone(), pool.shards()).await { + Some(s) => { + set_sharding_key(&mut self.write).await?; + shard = Some(s); + continue; + } + None => (), + }; // The message is part of the regular protocol. // self.buffer.put(message); // Grab a server from the pool. // None = any shard - let connection = pool.get(None).await.unwrap(); + let connection = pool.get(shard).await.unwrap(); let mut proxy = connection.0; let _address = connection.1; let server = &mut *proxy; @@ -230,6 +255,7 @@ impl Client { // Release server if !server.in_transaction() && self.transaction_mode { + shard = None; break; } } @@ -288,6 +314,7 @@ impl Client { // Release server if !server.in_transaction() && self.transaction_mode { + shard = None; break; } } @@ -314,6 +341,7 @@ impl Client { // Release the server if !server.in_transaction() && self.transaction_mode { println!("Releasing after copy done"); + shard = None; break; } } @@ -333,4 +361,30 @@ impl Client { let mut guard = self.client_server_map.lock().unwrap(); guard.remove(&(self.process_id, self.secret_key)); } + + async fn select_shard(&mut self, mut buf: BytesMut, shards: usize) -> Option { + let code = buf.get_u8() as char; + + match code { + 'Q' => (), + // 'P' => (), + _ => return None, + }; + + let len = buf.get_i32(); + let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); // Don't read the ternminating null + + if self.sharding_regex.is_match(&query) { + let shard = query.split("'").collect::>()[1]; + match shard.parse::() { + Ok(shard) => { + let sharder = Sharder::new(shards); + Some(sharder.pg_bigint_hash(shard)) + } + Err(_) => None, + } + } else { + None + } + } } diff --git a/src/messages.rs b/src/messages.rs index 58459f97e..eb65c90e1 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -138,6 +138,23 @@ pub async fn md5_password( Ok(write_all(stream, message).await?) } +pub async fn set_sharding_key(stream: &mut OwnedWriteHalf) -> Result<(), Error> { + let mut res = BytesMut::with_capacity(25); + + let set_complete = BytesMut::from(&"SET SHARDING KEY\0"[..]); + let len = (set_complete.len() + 4) as i32; + + res.put_u8(b'C'); + res.put_i32(len); + res.put_slice(&set_complete[..]); + + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} + /// Write all data in the buffer to the TcpStream. pub async fn write_all(stream: &mut TcpStream, buf: BytesMut) -> Result<(), Error> { match stream.write_all(&buf).await { From 9065a587e004c1474e17fc34c6d05f86d620064c Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 13:13:12 -0800 Subject: [PATCH 028/551] readme --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d7217e9c8..e0635281d 100644 --- a/README.md +++ b/README.md @@ -70,11 +70,18 @@ We're implemeting Postgres' `PARTITION BY HASH` sharding function for `BIGINT` f The biggest advantage of using this sharding function is that anyone can shard the dataset using Postgres partitions while also access it for both reads and writes using this pooler. No custom obscure sharding function is needed and database sharding can be done entirely in Postgres. +To select the shard we want to talk to, we introduced special syntax: + +```sql +SET SHARDING KEY TO '1234'; +``` + +This sharding key will be hashed and the pooler will select a shard to use for the next transaction. If the pooler is in session mode, this sharding key will be used until it's set again or the client disconnects. + ## Missing -1. Query routing based on shard. -2. Authentication, ehem, this proxy is letting anyone in at the moment. +1. Authentication, ehem, this proxy is letting anyone in at the moment. ## Benchmarks From a2dd0d761806ad2cd670dbeaa84800460243f658 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 13:40:46 -0800 Subject: [PATCH 029/551] readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e0635281d..653bae7ff 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su 1. Install Rust (latest stable is fine). 2. `cargo run --release` (to get better benchmarks). -3. Install Postgres and create a user and a DB, e.g. `CREATE ROLE lev ENCRYPTED PASSWORD 'lev' LOGIN;` and `createdb lev`. +3. Change the config in `pgcat.toml` to fit your setup. +4. Install Postgres and create a user and a DB, e.g. `CREATE ROLE lev ENCRYPTED PASSWORD 'lev' LOGIN;` and `createdb lev`. ### Tests From 122d5f8674040cb31cb247479fcd5ed77730fa05 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 14:56:57 -0800 Subject: [PATCH 030/551] Update LICENSE --- LICENSE | 875 +++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 674 insertions(+), 201 deletions(-) diff --git a/LICENSE b/LICENSE index 261eeb9e9..94a9ed024 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,674 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. From fe28a6957e46ae83008d9d85610d9ea1d96b0d97 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 14:59:10 -0800 Subject: [PATCH 031/551] Update main.rs --- src/main.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main.rs b/src/main.rs index 63972bfa1..61b4d66f0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,18 +1,18 @@ -// -// Copyright 2022 Lev Kokotov -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// +// PgCat, a PostgreSQL pooler with load balancing, failover, and sharding support. +// Copyright (C) 2022 Lev Kokotov + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . extern crate async_trait; extern crate bb8; extern crate bytes; From 9f098341f7e62fee7d817c69f33977d100204822 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 14:59:44 -0800 Subject: [PATCH 032/551] Update main.rs --- src/main.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main.rs b/src/main.rs index 61b4d66f0..8660086ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,18 +1,18 @@ -// PgCat, a PostgreSQL pooler with load balancing, failover, and sharding support. -// Copyright (C) 2022 Lev Kokotov +// PgCat, a PostgreSQL pooler with load balancing, failover, and sharding support. +// Copyright (C) 2022 Lev Kokotov -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// NU General Public License for more details. -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . extern crate async_trait; extern crate bb8; extern crate bytes; From c724f336a1b6c4847e1da04fc20389207d540160 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 15:00:16 -0800 Subject: [PATCH 033/551] Update main.rs --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 8660086ca..0602419cd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// NU General Public License for more details. +// GNU General Public License for more details. // You should have received a copy of the GNU General Public License // along with this program. If not, see . From a06f444afa0cdee6ae3c482726d8927c36e1af82 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 15:48:28 -0800 Subject: [PATCH 034/551] fixed health check; sharding setup and tests --- README.md | 2 +- pgcat.toml | 16 +++-- src/pool.rs | 18 +++++- ...etup.sql => partition_hash_test_setup.sql} | 0 tests/sharding/query_routing.sh | 12 ++++ tests/sharding/query_routing_setup.sql | 61 +++++++++++++++++++ tests/sharding/query_routing_test_insert.sql | 47 ++++++++++++++ tests/sharding/query_routing_test_select.sql | 47 ++++++++++++++ .../sharding/query_routing_test_validate.sql | 11 ++++ 9 files changed, 206 insertions(+), 8 deletions(-) rename tests/sharding/{setup.sql => partition_hash_test_setup.sql} (100%) create mode 100644 tests/sharding/query_routing.sh create mode 100644 tests/sharding/query_routing_setup.sql create mode 100644 tests/sharding/query_routing_test_insert.sql create mode 100644 tests/sharding/query_routing_test_select.sql create mode 100644 tests/sharding/query_routing_test_validate.sql diff --git a/README.md b/README.md index 653bae7ff..20d9dc079 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ either lose 1/x of your traffic or risk losing it all eventually. Ideally you ov to make this choice :-). ### Sharding -We're implemeting Postgres' `PARTITION BY HASH` sharding function for `BIGINT` fields. This works well for tables that use `BIGSERIAL` primary key which I think is common enough these days. We can also add many more functions here, but this is a good start. See `src/sharding.rs` and `tests/sharding/setup.sql` for more details on the implementation. +We're implemeting Postgres' `PARTITION BY HASH` sharding function for `BIGINT` fields. This works well for tables that use `BIGSERIAL` primary key which I think is common enough these days. We can also add many more functions here, but this is a good start. See `src/sharding.rs` and `tests/sharding/partition_hash_test_setup.sql` for more details on the implementation. The biggest advantage of using this sharding function is that anyone can shard the dataset using Postgres partitions while also access it for both reads and writes using this pooler. No custom obscure sharding function is needed and database sharding can be done entirely in Postgres. diff --git a/pgcat.toml b/pgcat.toml index bb410495b..78f557018 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -32,8 +32,8 @@ ban_time = 60 # Seconds # # User to use for authentication against the server. [user] -name = "lev" -password = "lev" +name = "sharding_user" +password = "sharding_user" # @@ -49,7 +49,7 @@ servers = [ [ "localhost", 5432 ], ] # Database name (e.g. "postgres") -database = "lev" +database = "shard0" [shards.1] # [ host, port ] @@ -57,4 +57,12 @@ servers = [ [ "127.0.0.1", 5432 ], [ "localhost", 5432 ], ] -database = "lev" \ No newline at end of file +database = "shard1" + +[shards.2] +# [ host, port ] +servers = [ + [ "127.0.0.1", 5432 ], + [ "localhost", 5432 ], +] +database = "shard2" \ No newline at end of file diff --git a/src/pool.rs b/src/pool.rs index 874fd615f..693b472d5 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -185,10 +185,22 @@ impl ConnectionPool { ) .await { - Ok(_) => return Ok((conn, address)), + // Check if health check succeeded + Ok(res) => match res { + Ok(_) => return Ok((conn, address)), + Err(_) => { + println!( + ">> Banning replica {} because of failed health check", + index + ); + self.ban(&address, shard); + continue; + } + }, + // Health check never came back, database is really really down Err(_) => { println!( - ">> Banning replica {} because of failed health check", + ">> Banning replica {} because of health check timeout", index ); self.ban(&address, shard); @@ -280,7 +292,7 @@ impl ManageConnection for ServerPool { /// Attempts to create a new connection. async fn connect(&self) -> Result { - println!(">> Getting new connection from the pool"); + println!(">> Creating a new connection for the pool"); Server::startup( &self.address.host, diff --git a/tests/sharding/setup.sql b/tests/sharding/partition_hash_test_setup.sql similarity index 100% rename from tests/sharding/setup.sql rename to tests/sharding/partition_hash_test_setup.sql diff --git a/tests/sharding/query_routing.sh b/tests/sharding/query_routing.sh new file mode 100644 index 000000000..d1b2b845a --- /dev/null +++ b/tests/sharding/query_routing.sh @@ -0,0 +1,12 @@ +#/bin/bash + +# Setup all the shards. +sudo service postgresql restart + +psql -f query_routing_setup.sql + +psql -h 127.0.0.1 -p 6432 -f query_routing_test_insert.sql + +psql -h 127.0.0.1 -p 6432 -f query_routing_test_select.sql + +psql -f query_routing_test_validate.sql \ No newline at end of file diff --git a/tests/sharding/query_routing_setup.sql b/tests/sharding/query_routing_setup.sql new file mode 100644 index 000000000..d4e766dab --- /dev/null +++ b/tests/sharding/query_routing_setup.sql @@ -0,0 +1,61 @@ + +DROP DATABASE IF EXISTS shard0; +DROP DATABASE IF EXISTS shard1; +DROP DATABASE IF EXISTS shard2; + +CREATE DATABASE shard0; +CREATE DATABASE shard1; +CREATE DATABASE shard2; + +\c shard0 + +DROP TABLE IF EXISTS data CASCADE; + +CREATE TABLE data ( + id BIGINT, + value VARCHAR +) PARTITION BY HASH (id); + +CREATE TABLE data_shard_0 PARTITION OF data FOR VALUES WITH (MODULUS 3, REMAINDER 0); + +\c shard1 + +DROP TABLE IF EXISTS data CASCADE; + +CREATE TABLE data ( + id BIGINT, + value VARCHAR +) PARTITION BY HASH (id); + +CREATE TABLE data_shard_1 PARTITION OF data FOR VALUES WITH (MODULUS 3, REMAINDER 1); + + +\c shard2 + +DROP TABLE IF EXISTS data CASCADE; + +CREATE TABLE data ( + id BIGINT, + value VARCHAR +) PARTITION BY HASH (id); + +CREATE TABLE data_shard_2 PARTITION OF data FOR VALUES WITH (MODULUS 3, REMAINDER 2); + +DROP ROLE IF EXISTS sharding_user; +CREATE ROLE sharding_user ENCRYPTED PASSWORD 'sharding_user' LOGIN; + +GRANT CONNECT ON DATABASE shard0 TO sharding_user; +GRANT CONNECT ON DATABASE shard1 TO sharding_user; +GRANT CONNECT ON DATABASE shard2 TO sharding_user; + +\c shard0 +GRANT ALL ON SCHEMA public TO sharding_user; +GRANT ALL ON TABLE data TO sharding_user; + +\c shard1 +GRANT ALL ON SCHEMA public TO sharding_user; +GRANT ALL ON TABLE data TO sharding_user; + +\c shard2 +GRANT ALL ON SCHEMA public TO sharding_user; +GRANT ALL ON TABLE data TO sharding_user; \ No newline at end of file diff --git a/tests/sharding/query_routing_test_insert.sql b/tests/sharding/query_routing_test_insert.sql new file mode 100644 index 000000000..97d5babd6 --- /dev/null +++ b/tests/sharding/query_routing_test_insert.sql @@ -0,0 +1,47 @@ +SET SHARDING KEY TO '1'; +INSERT INTO data (id, value) VALUES (1, 'value_1'); + +SET SHARDING KEY TO '2'; +INSERT INTO data (id, value) VALUES (2, 'value_1'); + +SET SHARDING KEY TO '3'; +INSERT INTO data (id, value) VALUES (3, 'value_1'); + +SET SHARDING KEY TO '4'; +INSERT INTO data (id, value) VALUES (4, 'value_1'); + +SET SHARDING KEY TO '5'; +INSERT INTO data (id, value) VALUES (5, 'value_1'); + +SET SHARDING KEY TO '6'; +INSERT INTO data (id, value) VALUES (6, 'value_1'); + +SET SHARDING KEY TO '7'; +INSERT INTO data (id, value) VALUES (7, 'value_1'); + +SET SHARDING KEY TO '8'; +INSERT INTO data (id, value) VALUES (8, 'value_1'); + +SET SHARDING KEY TO '9'; +INSERT INTO data (id, value) VALUES (9, 'value_1'); + +SET SHARDING KEY TO '10'; +INSERT INTO data (id, value) VALUES (10, 'value_1'); + +SET SHARDING KEY TO '11'; +INSERT INTO data (id, value) VALUES (11, 'value_1'); + +SET SHARDING KEY TO '12'; +INSERT INTO data (id, value) VALUES (12, 'value_1'); + +SET SHARDING KEY TO '13'; +INSERT INTO data (id, value) VALUES (13, 'value_1'); + +SET SHARDING KEY TO '14'; +INSERT INTO data (id, value) VALUES (14, 'value_1'); + +SET SHARDING KEY TO '15'; +INSERT INTO data (id, value) VALUES (15, 'value_1'); + +SET SHARDING KEY TO '16'; +INSERT INTO data (id, value) VALUES (16, 'value_1'); \ No newline at end of file diff --git a/tests/sharding/query_routing_test_select.sql b/tests/sharding/query_routing_test_select.sql new file mode 100644 index 000000000..c5778034e --- /dev/null +++ b/tests/sharding/query_routing_test_select.sql @@ -0,0 +1,47 @@ +SET SHARDING KEY TO '1'; +SELECT * FROM data WHERE id = 1; + +SET SHARDING KEY TO '2'; +SELECT * FROM data WHERE id = 2; + +SET SHARDING KEY TO '3'; +SELECT * FROM data WHERE id = 3; + +SET SHARDING KEY TO '4'; +SELECT * FROM data WHERE id = 4; + +SET SHARDING KEY TO '5'; +SELECT * FROM data WHERE id = 5; + +SET SHARDING KEY TO '6'; +SELECT * FROM data WHERE id = 6; + +SET SHARDING KEY TO '7'; +SELECT * FROM data WHERE id = 7; + +SET SHARDING KEY TO '8'; +SELECT * FROM data WHERE id = 8; + +SET SHARDING KEY TO '9'; +SELECT * FROM data WHERE id = 9; + +SET SHARDING KEY TO '10'; +SELECT * FROM data WHERE id = 10; + +SET SHARDING KEY TO '11'; +SELECT * FROM data WHERE id = 11; + +SET SHARDING KEY TO '12'; +SELECT * FROM data WHERE id = 12; + +SET SHARDING KEY TO '13'; +SELECT * FROM data WHERE id = 13; + +SET SHARDING KEY TO '14'; +SELECT * FROM data WHERE id = 14; + +SET SHARDING KEY TO '15'; +SELECT * FROM data WHERE id = 15; + +SET SHARDING KEY TO '16'; +SELECT * FROM data WHERE id = 16; \ No newline at end of file diff --git a/tests/sharding/query_routing_test_validate.sql b/tests/sharding/query_routing_test_validate.sql new file mode 100644 index 000000000..5ef9a5686 --- /dev/null +++ b/tests/sharding/query_routing_test_validate.sql @@ -0,0 +1,11 @@ +\c shard0 + +SELECT * FROM data; + +\c shard1 + +SELECT * FROM data; + +\c shard2 + +SELECT * FROM data; \ No newline at end of file From 561e48a84cf5ac9046d680021945aad429ad7229 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 16:56:29 -0800 Subject: [PATCH 035/551] health check timeout config --- src/pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pool.rs b/src/pool.rs index 693b472d5..ed267ed29 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -180,7 +180,7 @@ impl ConnectionPool { let server = &mut *conn; match tokio::time::timeout( - tokio::time::Duration::from_millis(HEALTHCHECK_TIMEOUT), + tokio::time::Duration::from_millis(self.healthcheck_timeout), server.query("SELECT 1"), ) .await From 8fc5ebdaf607b030111d5c91430cea8fc7f251d6 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 17:08:17 -0800 Subject: [PATCH 036/551] warnings --- Cargo.toml | 2 +- src/client.rs | 1 - src/config.rs | 27 +++++++++++++------------ src/errors.rs | 6 +++--- src/main.rs | 2 +- src/pool.rs | 52 +------------------------------------------------ src/sharding.rs | 2 +- 7 files changed, 22 insertions(+), 70 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 019c15391..d86619823 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,4 +17,4 @@ sha-1 = "*" toml = "*" serde = "*" serde_derive = "*" -regex = "1" \ No newline at end of file +regex = "1" diff --git a/src/client.rs b/src/client.rs index 0ba97d0c0..31b61ff0c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,7 +2,6 @@ /// We are pretending to the server in this scenario, /// and this module implements that. use bytes::{Buf, BufMut, BytesMut}; -use rand::{distributions::Alphanumeric, Rng}; use regex::Regex; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; diff --git a/src/config.rs b/src/config.rs index bfc5619ad..ed897b22b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,10 +1,10 @@ use serde_derive::Deserialize; -use std::collections::HashMap; -use std::path::Path; use tokio::fs::File; use tokio::io::AsyncReadExt; use toml; +use std::collections::HashMap; + use crate::errors::Error; #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)] @@ -62,16 +62,6 @@ pub async fn parse(path: &str) -> Result { } }; - // let config: toml::Value = match toml::from_str(&contents) { - // Ok(config) => config, - // Err(err) => { - // println!("> Config error: {:?}", err); - // return Err(Error::BadConfig); - // } - // }; - - // println!("Config: {:?}", config); - let config: Config = match toml::from_str(&contents) { Ok(config) => config, Err(err) => { @@ -82,3 +72,16 @@ pub async fn parse(path: &str) -> Result { Ok(config) } + +#[cfg(test)] +mod test { + use super::*; + + #[tokio::test] + async fn test_config() { + let config = parse("pgcat.toml").await.unwrap(); + assert_eq!(config.general.pool_size, 15); + assert_eq!(config.shards.len(), 3); + assert_eq!(config.shards["1"].servers[0].0, "127.0.0.1"); + } +} diff --git a/src/errors.rs b/src/errors.rs index 5ee43e3f0..3dcbf74b1 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,11 +1,11 @@ #[derive(Debug, PartialEq)] pub enum Error { SocketError, - ClientDisconnected, + // ClientDisconnected, ClientBadStartup, ProtocolSyncError, ServerError, - ServerTimeout, - DirtyServer, + // ServerTimeout, + // DirtyServer, BadConfig, } diff --git a/src/main.rs b/src/main.rs index 0602419cd..9b084ecbe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,7 +37,6 @@ mod sharding; // Support for query cancellation: this maps our process_ids and // secret keys to the backend's. -use config::{Address, User}; use pool::{ClientServerMap, ConnectionPool}; /// Main! @@ -48,6 +47,7 @@ async fn main() { let config = match config::parse("pgcat.toml").await { Ok(config) => config, Err(err) => { + println!("> Config parse error: {:?}", err); return; } }; diff --git a/src/pool.rs b/src/pool.rs index ed267ed29..eb0681675 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -18,19 +18,6 @@ pub type BanList = Arc>>>; pub type Counter = Arc; pub type ClientServerMap = Arc>>; -// 60 seconds of ban time. -// After that, the replica will be allowed to serve traffic again. -const BAN_TIME: i64 = 60; - -// DB pool size (per actual database server) -const POOL_SIZE: u32 = 15; - -// 5 seconds to connect before we give up -const CONNECT_TIMEOUT: u64 = 5000; - -// How much time to give the server to answer a SELECT 1 query. -const HEALTHCHECK_TIMEOUT: u64 = 1000; - #[derive(Clone, Debug)] pub struct ConnectionPool { databases: Vec>>, @@ -42,43 +29,6 @@ pub struct ConnectionPool { } impl ConnectionPool { - // Construct the connection pool for a single-shard cluster. - pub async fn new( - addresses: Vec
, - user: User, - database: &str, - client_server_map: ClientServerMap, - ) -> ConnectionPool { - let mut databases = Vec::new(); - - for address in &addresses { - let manager = ServerPool::new( - address.clone(), - user.clone(), - database, - client_server_map.clone(), - ); - let pool = Pool::builder() - .max_size(POOL_SIZE) - .connection_timeout(std::time::Duration::from_millis(CONNECT_TIMEOUT)) - .test_on_check_out(false) - .build(manager) - .await - .unwrap(); - - databases.push(pool); - } - - ConnectionPool { - databases: vec![databases], - addresses: vec![addresses], - round_robin: Arc::new(AtomicUsize::new(0)), - banlist: Arc::new(Mutex::new(vec![HashMap::new()])), - healthcheck_timeout: HEALTHCHECK_TIMEOUT, - ban_time: BAN_TIME, - } - } - /// Construct the connection pool from a config file. pub async fn from_config(config: Config, client_server_map: ClientServerMap) -> ConnectionPool { let mut shards = Vec::new(); @@ -222,7 +172,7 @@ impl ConnectionPool { /// Clear the replica to receive traffic again. Takes effect immediately /// for all new transactions. - pub fn unban(&self, address: &Address, shard: usize) { + pub fn _unban(&self, address: &Address, shard: usize) { let mut guard = self.banlist.lock().unwrap(); guard[shard].remove(address); } diff --git a/src/sharding.rs b/src/sharding.rs index b72923491..3f32dbe97 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -14,7 +14,7 @@ impl Sharder { /// Use SHA1 to pick a shard for the key. The key can be anything, /// including an int or a string. - pub fn sha1(&self, key: &[u8]) -> usize { + pub fn _sha1(&self, key: &[u8]) -> usize { let mut hasher = Sha1::new(); hasher.update(key); let result = hasher.finalize_reset(); From d047b746f5e64d3e43a55bb1a85c3621f5362ec9 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 17:15:35 -0800 Subject: [PATCH 037/551] sharding readme --- src/sharding.rs | 2 +- tests/sharding/README.md | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/sharding/README.md diff --git a/src/sharding.rs b/src/sharding.rs index 3f32dbe97..7e6abc447 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -125,7 +125,7 @@ mod test { assert_eq!(shard, 1); } - // See tests/sharding/setup.sql + // See tests/sharding/partition_hash_test_setup.sql // The output of those SELECT statements will match this test, // confirming that we implemented Postgres BIGINT hashing correctly. #[test] diff --git a/tests/sharding/README.md b/tests/sharding/README.md new file mode 100644 index 000000000..abb908dcb --- /dev/null +++ b/tests/sharding/README.md @@ -0,0 +1,35 @@ +# Sharding tests + +This helps us test the sharding algorithm we implemented. + + +## Setup + +We setup 3 Postgres DBs, `shard0`, `shard1`, and `shard2`. In each database, we create a partitioned table called `data`. The table is partitioned by hash, and each database will only have _one_ partition, `shard0` will satisfy `modulus 3, remainder 0`, `shard1` will satisfy `modulus 3, remainder 1`, etc. + +To set this up, you can just run: + +```bash +psql -f query_routing_setup.sql +``` + +## Run the tests + +Start up PgCat by running `cargo run --release` in the root of the repo. In a different tab, run this: + +```bash +psql -h 127.0.0.1 -p 6432 -f query_routing_test_insert.sql +psql -h 127.0.0.1 -p 6432 -f query_routing_test_select.sql +``` + +Note that no errors should take place. If our sharding logic was incorrect, we would get some errors +about unsatisfiable partition bounds. We don't because the pooler picked the correct databases +given the sharding keys. + +Finally, you can validate the result again by running + +```bash +psql -f query_routing_test_validate.sql +``` + +## That's it! \ No newline at end of file From 8a443719885714ad69db8f644b3ab825e49cc5f8 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 17:16:23 -0800 Subject: [PATCH 038/551] links --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 20d9dc079..f62ad7612 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && \ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended ``` +See [sharding README](./tests/sharding/README.md) for sharding logic testing. + ## Features 1. Session mode. From b154b09bed23ba66643e639f96f8c4703a146c78 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 17:19:12 -0800 Subject: [PATCH 039/551] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f62ad7612..f7bd9cc10 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su 1. Install Rust (latest stable is fine). 2. `cargo run --release` (to get better benchmarks). 3. Change the config in `pgcat.toml` to fit your setup. -4. Install Postgres and create a user and a DB, e.g. `CREATE ROLE lev ENCRYPTED PASSWORD 'lev' LOGIN;` and `createdb lev`. +4. Install Postgres and run `psql -f tests/sharding/query_routing_setup.sql` ### Tests From e53cab5938f52a7dbc78f6d357da279c744dee7f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 17:19:38 -0800 Subject: [PATCH 040/551] readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7bd9cc10..5fe52e927 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su 1. Install Rust (latest stable is fine). 2. `cargo run --release` (to get better benchmarks). -3. Change the config in `pgcat.toml` to fit your setup. +3. Change the config in `pgcat.toml` to fit your setup (optional given next step). 4. Install Postgres and run `psql -f tests/sharding/query_routing_setup.sql` ### Tests From 8394b8cc0bc34424cd5b43ab8b6f74f8e23c98e4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 17:20:31 -0800 Subject: [PATCH 041/551] readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5fe52e927..8e635452e 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ See [sharding README](./tests/sharding/README.md) for sharding logic testing. 4. Query cancellation. 5. Round-robin load balancing of replicas. 6. Banlist & failover +7. Sharding! ### Session mode Each client owns its own server for the duration of the session. Commands like `SET` are allowed. From c4f0d6c4663709b3f1ebd44e60b3d9c8336d212f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 17:27:54 -0800 Subject: [PATCH 042/551] readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e635452e..8269ca4ea 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su ## Local development -1. Install Rust (latest stable is fine). +1. Install Rust (latest stable will work great). 2. `cargo run --release` (to get better benchmarks). 3. Change the config in `pgcat.toml` to fit your setup (optional given next step). 4. Install Postgres and run `psql -f tests/sharding/query_routing_setup.sql` From e055834d8370bfcfbb4c6571a132f012304abcb5 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 17:40:28 -0800 Subject: [PATCH 043/551] remove unused --- src/sharding.rs | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/sharding.rs b/src/sharding.rs index 7e6abc447..1bbb6d879 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -1,5 +1,3 @@ -use sha1::{Digest, Sha1}; - // https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/include/catalog/partition.h#L20 const PARTITION_HASH_SEED: u64 = 0x7A5B22367996DCFD; @@ -12,17 +10,6 @@ impl Sharder { Sharder { shards: shards } } - /// Use SHA1 to pick a shard for the key. The key can be anything, - /// including an int or a string. - pub fn _sha1(&self, key: &[u8]) -> usize { - let mut hasher = Sha1::new(); - hasher.update(key); - let result = hasher.finalize_reset(); - - let i = u32::from_le_bytes(result[result.len() - 4..result.len()].try_into().unwrap()); - i as usize % self.shards - } - /// Hash function used by Postgres to determine which partition /// to put the row in when using HASH(column) partitioning. /// Source: https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/common/hashfn.c#L631 @@ -117,14 +104,6 @@ impl Sharder { mod test { use super::*; - #[test] - fn test_sha1() { - let sharder = Sharder::new(12); - let key = b"1234"; - let shard = sharder.sha1(key); - assert_eq!(shard, 1); - } - // See tests/sharding/partition_hash_test_setup.sql // The output of those SELECT statements will match this test, // confirming that we implemented Postgres BIGINT hashing correctly. From f74b3e55d7711e943be35202fddf4624a77cde60 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 17:58:44 -0800 Subject: [PATCH 044/551] Create CONTRIBUTING.md --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..af38a647f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,6 @@ +Thank you for contributing! Just a few tips here: + +1. `cargo fmt` your code before opening up a PR +2. Run the "test suite" (i.e. PgBench) to make sure everything still works. + +Happy hacking! From 81793de20ce7289f77b24d2e2a8ca11ea6e90618 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 18:02:26 -0800 Subject: [PATCH 045/551] rebuild --- .circleci/config.yml | 9 ++++++--- Cargo.toml | 16 ++++++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bb7d289bb..35826ec7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,12 +15,15 @@ jobs: steps: - checkout - restore_cache: - key: cargo-lock-{{ checksum "Cargo.lock" }} + key: cargo-lock-2-{{ checksum "Cargo.lock" }} - run: name: "Build" - command: "cargo build --release" + command: "cargo build" + - run: + name: "Test" + command: "cargo test" - save_cache: - key: cargo-lock-{{ checksum "Cargo.lock" }} + key: cargo-lock-2-{{ checksum "Cargo.lock" }} paths: - target - ~/.cargo diff --git a/Cargo.toml b/Cargo.toml index d86619823..fd66048c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,13 +8,13 @@ edition = "2021" [dependencies] tokio = { version = "1", features = ["full"] } bytes = "1" -md-5 = "*" -bb8 = "*" -async-trait = "*" -rand = "*" +md-5 = "0.10" +bb8 = "0.7" +async-trait = "0.1" +rand = "0.8" chrono = "0.4" -sha-1 = "*" -toml = "*" -serde = "*" -serde_derive = "*" +sha-1 = "0.10" +toml = "0.5" +serde = "1" +serde_derive = "1" regex = "1" From f26fa925c2583e9e456dfae382f85b6f28528388 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Feb 2022 21:59:56 -0800 Subject: [PATCH 046/551] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8269ca4ea..25e213d8e 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ To select the shard we want to talk to, we introduced special syntax: SET SHARDING KEY TO '1234'; ``` -This sharding key will be hashed and the pooler will select a shard to use for the next transaction. If the pooler is in session mode, this sharding key will be used until it's set again or the client disconnects. +This sharding key will be hashed and the pooler will select a shard to use for the next transaction. If the pooler is in session mode, this sharding key has to be set as the first query on startup & cannot be changed until the client re-connects. ## Missing @@ -205,4 +205,4 @@ number of transactions actually processed: 1000/1000 latency average = 0.931 ms tps = 1074.017747 (including connections establishing) tps = 1077.121752 (excluding connections establishing) -``` \ No newline at end of file +``` From 64460b88c2d1c63f169edf88e9bf41cfd2a783d5 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 9 Feb 2022 06:51:31 -0800 Subject: [PATCH 047/551] some comments --- src/client.rs | 16 ++++++++++------ src/config.rs | 2 +- src/messages.rs | 7 +++++++ src/pool.rs | 4 ++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/client.rs b/src/client.rs index 31b61ff0c..85cfbe9d8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -168,7 +168,7 @@ impl Client { // Active shard we're talking to. // The lifetime of this depends on the pool mode: - // - if in session mode, this lives until client disconnects or changes it, + // - if in session mode, this lives until the client disconnects, // - if in transaction mode, this lives for the duration of one transaction. let mut shard: Option = None; @@ -177,7 +177,7 @@ impl Client { // either a `Q` (query) or `P` (prepare, extended protocol). // We can parse it here before grabbing a server from the pool, // in case the client is sending some control messages, e.g. - // SET sharding_context.key = '1234'; + // SET SHARDING KEY TO 'bigint'; let mut message = read_message(&mut self.read).await?; // Parse for special select shard command. @@ -191,9 +191,6 @@ impl Client { None => (), }; - // The message is part of the regular protocol. - // self.buffer.put(message); - // Grab a server from the pool. // None = any shard let connection = pool.get(shard).await.unwrap(); @@ -361,12 +358,19 @@ impl Client { guard.remove(&(self.process_id, self.secret_key)); } + /// Determine if the query is part of our special syntax, extract + /// the shard key, and return the shard to query based on Postgres' + /// PARTITION BY HASH function. async fn select_shard(&mut self, mut buf: BytesMut, shards: usize) -> Option { let code = buf.get_u8() as char; + // Only supporting simpe protocol here, so + // one would have to execute something like this: + // psql -c "SET SHARDING KEY TO '1234'" + // after sanitizing the value manually, which can be just done with an + // int parser, e.g. `let key = "1234".parse::().unwrap()`. match code { 'Q' => (), - // 'P' => (), _ => return None, }; diff --git a/src/config.rs b/src/config.rs index ed897b22b..39218f751 100644 --- a/src/config.rs +++ b/src/config.rs @@ -43,8 +43,8 @@ pub struct Config { pub shards: HashMap, } +/// Parse the config. pub async fn parse(path: &str) -> Result { - // let path = Path::new(path); let mut contents = String::new(); let mut file = match File::open(path).await { Ok(file) => file, diff --git a/src/messages.rs b/src/messages.rs index eb65c90e1..90a670048 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -26,6 +26,9 @@ pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { /// Send server parameters to the client. This will tell the client /// what server version and what's the encoding we're using. +// +// TODO: Forward these from the server instead of hardcoding. +// pub async fn server_parameters(stream: &mut TcpStream) -> Result<(), Error> { let client_encoding = BytesMut::from(&b"client_encoding\0UTF8\0"[..]); let server_version = @@ -138,16 +141,20 @@ pub async fn md5_password( Ok(write_all(stream, message).await?) } +/// Implements a response to our custom `SET SHARDING KEY` command. +/// This tells the client we're ready for the next query. pub async fn set_sharding_key(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let mut res = BytesMut::with_capacity(25); let set_complete = BytesMut::from(&"SET SHARDING KEY\0"[..]); let len = (set_complete.len() + 4) as i32; + // CommandComplete res.put_u8(b'C'); res.put_i32(len); res.put_slice(&set_complete[..]); + // ReadyForQuery (idle) res.put_u8(b'Z'); res.put_i32(5); res.put_u8(b'I'); diff --git a/src/pool.rs b/src/pool.rs index eb0681675..624818a10 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -182,7 +182,7 @@ impl ConnectionPool { pub fn is_banned(&self, address: &Address, shard: usize) -> bool { let mut guard = self.banlist.lock().unwrap(); - // Everything is banned, nothig is banned + // Everything is banned = nothing is banned. if guard[shard].len() == self.databases[shard].len() { guard[shard].clear(); drop(guard); @@ -194,8 +194,8 @@ impl ConnectionPool { match guard[shard].get(address) { Some(timestamp) => { let now = chrono::offset::Utc::now().naive_utc(); + // Ban expired. if now.timestamp() - timestamp.timestamp() > self.ban_time { - // 1 minute guard[shard].remove(address); false } else { From 3b98f16c83716c026ff6a46f26309d53b4d36f7f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 9 Feb 2022 20:02:20 -0800 Subject: [PATCH 048/551] #1 Primary/replica selection --- pgcat.toml | 20 ++++++++--------- src/client.rs | 59 +++++++++++++++++++++++++++++++++++++++++++++---- src/config.rs | 10 ++++++++- src/messages.rs | 10 ++++++--- src/pool.rs | 28 ++++++++++++++++++++++- src/server.rs | 7 +++++- 6 files changed, 114 insertions(+), 20 deletions(-) diff --git a/pgcat.toml b/pgcat.toml index 78f557018..803a3422b 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -43,26 +43,26 @@ password = "sharding_user" # Shard 0 [shards.0] -# [ host, port ] +# [ host, port, role ] servers = [ - [ "127.0.0.1", 5432 ], - [ "localhost", 5432 ], + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ], ] # Database name (e.g. "postgres") database = "shard0" [shards.1] -# [ host, port ] +# [ host, port, role ] servers = [ - [ "127.0.0.1", 5432 ], - [ "localhost", 5432 ], + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ], ] database = "shard1" [shards.2] -# [ host, port ] +# [ host, port, role ] servers = [ - [ "127.0.0.1", 5432 ], - [ "localhost", 5432 ], + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ], ] -database = "shard2" \ No newline at end of file +database = "shard2" diff --git a/src/client.rs b/src/client.rs index 85cfbe9d8..5df607796 100644 --- a/src/client.rs +++ b/src/client.rs @@ -7,6 +7,7 @@ use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::net::TcpStream; +use crate::config::Role; use crate::errors::Error; use crate::messages::*; use crate::pool::{ClientServerMap, ConnectionPool}; @@ -14,6 +15,7 @@ use crate::server::Server; use crate::sharding::Sharder; const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+';"; +const ROLE_REGEX: &str = r"SET SERVER ROLE TO '(PRIMARY|REPLICA)';"; /// The client state. One of these is created per client. pub struct Client { @@ -45,6 +47,9 @@ pub struct Client { // sharding regex sharding_regex: Regex, + + // role detection regex + role_regex: Regex, } impl Client { @@ -57,6 +62,7 @@ impl Client { transaction_mode: bool, ) -> Result { let sharding_regex = Regex::new(SHARDING_REGEX).unwrap(); + let role_regex = Regex::new(ROLE_REGEX).unwrap(); loop { // Could be StartupMessage or SSLRequest @@ -114,6 +120,7 @@ impl Client { secret_key: secret_key, client_server_map: client_server_map, sharding_regex: sharding_regex, + role_regex: role_regex, }); } @@ -134,6 +141,7 @@ impl Client { secret_key: secret_key, client_server_map: client_server_map, sharding_regex: sharding_regex, + role_regex: role_regex, }); } @@ -172,6 +180,8 @@ impl Client { // - if in transaction mode, this lives for the duration of one transaction. let mut shard: Option = None; + let mut role: Option = None; + loop { // Read a complete message from the client, which normally would be // either a `Q` (query) or `P` (prepare, extended protocol). @@ -182,18 +192,29 @@ impl Client { // Parse for special select shard command. // SET SHARDING KEY TO 'bigint'; - match self.select_shard(message.clone(), pool.shards()).await { + match self.select_shard(message.clone(), pool.shards()) { Some(s) => { - set_sharding_key(&mut self.write).await?; + custom_protocol_response_ok(&mut self.write, "SET SHARDING KEY").await?; shard = Some(s); continue; } None => (), }; + // Parse for special server role selection command. + // + match self.select_role(message.clone()) { + Some(r) => { + custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; + role = Some(r); + continue; + } + None => (), + }; + // Grab a server from the pool. // None = any shard - let connection = pool.get(shard).await.unwrap(); + let connection = pool.get(shard, role).await.unwrap(); let mut proxy = connection.0; let _address = connection.1; let server = &mut *proxy; @@ -252,6 +273,7 @@ impl Client { // Release server if !server.in_transaction() && self.transaction_mode { shard = None; + role = None; break; } } @@ -311,6 +333,7 @@ impl Client { // Release server if !server.in_transaction() && self.transaction_mode { shard = None; + role = None; break; } } @@ -338,6 +361,7 @@ impl Client { if !server.in_transaction() && self.transaction_mode { println!("Releasing after copy done"); shard = None; + role = None; break; } } @@ -361,7 +385,7 @@ impl Client { /// Determine if the query is part of our special syntax, extract /// the shard key, and return the shard to query based on Postgres' /// PARTITION BY HASH function. - async fn select_shard(&mut self, mut buf: BytesMut, shards: usize) -> Option { + fn select_shard(&mut self, mut buf: BytesMut, shards: usize) -> Option { let code = buf.get_u8() as char; // Only supporting simpe protocol here, so @@ -390,4 +414,31 @@ impl Client { None } } + + // Pick a primary or a replica from the pool. + fn select_role(&mut self, mut buf: BytesMut) -> Option { + let code = buf.get_u8() as char; + + // Same story as select_shard() above. + match code { + 'Q' => (), + _ => return None, + }; + + let len = buf.get_i32(); + let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); + + // Copy / paste from above. If we get one more of these use cases, + // it'll be time to abstract :). + if self.role_regex.is_match(&query) { + let role = query.split("'").collect::>()[1]; + match role { + "PRIMARY" => Some(Role::Primary), + "REPLICA" => Some(Role::Replica), + _ => return None, + } + } else { + None + } + } } diff --git a/src/config.rs b/src/config.rs index 39218f751..094fd7951 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,10 +7,17 @@ use std::collections::HashMap; use crate::errors::Error; +#[derive(Clone, PartialEq, Deserialize, Hash, std::cmp::Eq, Debug, Copy)] +pub enum Role { + Primary, + Replica, +} + #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)] pub struct Address { pub host: String, pub port: String, + pub role: Role, } #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Deserialize, Debug)] @@ -32,7 +39,7 @@ pub struct General { #[derive(Deserialize, Debug, Clone)] pub struct Shard { - pub servers: Vec<(String, u16)>, + pub servers: Vec<(String, u16, String)>, pub database: String, } @@ -83,5 +90,6 @@ mod test { assert_eq!(config.general.pool_size, 15); assert_eq!(config.shards.len(), 3); assert_eq!(config.shards["1"].servers[0].0, "127.0.0.1"); + assert_eq!(config.shards["0"].servers[0].2, "primary"); } } diff --git a/src/messages.rs b/src/messages.rs index 90a670048..5f17d8d5d 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -141,12 +141,16 @@ pub async fn md5_password( Ok(write_all(stream, message).await?) } -/// Implements a response to our custom `SET SHARDING KEY` command. +/// Implements a response to our custom `SET SHARDING KEY` +/// and `SET SERVER ROLE` commands. /// This tells the client we're ready for the next query. -pub async fn set_sharding_key(stream: &mut OwnedWriteHalf) -> Result<(), Error> { +pub async fn custom_protocol_response_ok( + stream: &mut OwnedWriteHalf, + message: &str, +) -> Result<(), Error> { let mut res = BytesMut::with_capacity(25); - let set_complete = BytesMut::from(&"SET SHARDING KEY\0"[..]); + let set_complete = BytesMut::from(&format!("{}\0", message)[..]); let len = (set_complete.len() + 4) as i32; // CommandComplete diff --git a/src/pool.rs b/src/pool.rs index 624818a10..57bc06623 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection}; use chrono::naive::NaiveDateTime; -use crate::config::{Address, Config, User}; +use crate::config::{Address, Config, Role, User}; use crate::errors::Error; use crate::server::Server; @@ -48,9 +48,19 @@ impl ConnectionPool { let mut replica_addresses = Vec::new(); for server in &shard.servers { + let role = match server.2.as_ref() { + "primary" => Role::Primary, + "replica" => Role::Replica, + _ => { + println!("> Config error: server role can be 'primary' or 'replica', have: '{}'. Defaulting to 'replica'.", server.2); + Role::Replica + } + }; + let address = Address { host: server.0.clone(), port: server.1.to_string(), + role: role, }; let manager = ServerPool::new( @@ -93,6 +103,7 @@ impl ConnectionPool { pub async fn get( &self, shard: Option, + role: Option, ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { // Set this to false to gain ~3-4% speed. let with_health_check = true; @@ -103,6 +114,9 @@ impl ConnectionPool { }; loop { + // TODO: think about making this local, so multiple clients + // don't compete for the same round-robin integer. + // Especially since we're going to be skipping (see role selection below). let index = self.round_robin.fetch_add(1, Ordering::SeqCst) % self.databases[shard].len(); let address = self.addresses[shard][index].clone(); @@ -111,6 +125,17 @@ impl ConnectionPool { continue; } + // Make sure you're getting a primary or a replica + // as per request. + match role { + Some(role) => { + if address.role != role { + continue; + } + } + None => (), + }; + // Check if we can connect // TODO: implement query wait timeout, i.e. time to get a conn from the pool let mut conn = match self.databases[shard][index].get().await { @@ -251,6 +276,7 @@ impl ManageConnection for ServerPool { &self.user.password, &self.database, self.client_server_map.clone(), + self.address.role, ) .await } diff --git a/src/server.rs b/src/server.rs index 0a7e31fc6..13f362a5b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -8,7 +8,7 @@ use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::net::TcpStream; -use crate::config::Address; +use crate::config::{Address, Role}; use crate::errors::Error; use crate::messages::*; use crate::ClientServerMap; @@ -48,6 +48,8 @@ pub struct Server { // Mapping of clients and servers used for query cancellation. client_server_map: ClientServerMap, + + role: Role, } impl Server { @@ -60,6 +62,7 @@ impl Server { password: &str, database: &str, client_server_map: ClientServerMap, + role: Role, ) -> Result { let mut stream = match TcpStream::connect(&format!("{}:{}", host, port)).await { Ok(stream) => stream, @@ -189,6 +192,7 @@ impl Server { data_available: false, bad: false, client_server_map: client_server_map, + role: role, }); } @@ -409,6 +413,7 @@ impl Server { Address { host: self.host.to_string(), port: self.port.to_string(), + role: self.role, } } } From ba1c165b24a04425ed8502e925eb55ec1f2d20c0 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 9 Feb 2022 21:19:14 -0800 Subject: [PATCH 049/551] fixes to the banlist --- pgcat.toml | 3 ++ src/client.rs | 15 ++++++- src/config.rs | 17 +++++++- src/errors.rs | 1 + src/pool.rs | 39 +++++++++++++++---- tests/sharding/query_routing.sh | 9 ++++- .../query_routing_test_primary_replica.sql | 13 +++++++ 7 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 tests/sharding/query_routing_test_primary_replica.sql diff --git a/pgcat.toml b/pgcat.toml index 803a3422b..ffcf7220d 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -47,6 +47,7 @@ password = "sharding_user" servers = [ [ "127.0.0.1", 5432, "primary" ], [ "localhost", 5432, "replica" ], + # [ "127.0.1.1", 5432, "replica" ], ] # Database name (e.g. "postgres") database = "shard0" @@ -56,6 +57,7 @@ database = "shard0" servers = [ [ "127.0.0.1", 5432, "primary" ], [ "localhost", 5432, "replica" ], + # [ "127.0.1.1", 5432, "replica" ], ] database = "shard1" @@ -64,5 +66,6 @@ database = "shard1" servers = [ [ "127.0.0.1", 5432, "primary" ], [ "localhost", 5432, "replica" ], + # [ "127.0.1.1", 5432, "replica" ], ] database = "shard2" diff --git a/src/client.rs b/src/client.rs index 5df607796..7d7e27a21 100644 --- a/src/client.rs +++ b/src/client.rs @@ -214,7 +214,14 @@ impl Client { // Grab a server from the pool. // None = any shard - let connection = pool.get(shard, role).await.unwrap(); + let connection = match pool.get(shard, role).await { + Ok(conn) => conn, + Err(err) => { + println!(">> Could not get connection from pool: {:?}", err); + return Err(err); + } + }; + let mut proxy = connection.0; let _address = connection.1; let server = &mut *proxy; @@ -253,10 +260,13 @@ impl Client { match code { 'Q' => { + // TODO: implement retries here for read-only transactions. server.send(original).await?; loop { + // TODO: implement retries here for read-only transactions. let response = server.recv().await?; + match write_all_half(&mut self.write, response).await { Ok(_) => (), Err(err) => { @@ -312,10 +322,13 @@ impl Client { 'S' => { // Extended protocol, client requests sync self.buffer.put(&original[..]); + + // TODO: retries for read-only transactions server.send(self.buffer.clone()).await?; self.buffer.clear(); loop { + // TODO: retries for read-only transactions let response = server.recv().await?; match write_all_half(&mut self.write, response).await { Ok(_) => (), diff --git a/src/config.rs b/src/config.rs index 094fd7951..fe9206d84 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,7 +3,7 @@ use tokio::fs::File; use tokio::io::AsyncReadExt; use toml; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use crate::errors::Error; @@ -77,6 +77,21 @@ pub async fn parse(path: &str) -> Result { } }; + // We use addresses as unique identifiers, + // let's make sure they are unique in the config as well. + for shard in &config.shards { + let mut dup_check = HashSet::new(); + + for server in &shard.1.servers { + dup_check.insert(server); + } + + if dup_check.len() != shard.1.servers.len() { + println!("> Shard {} contains duplicate server configs.", &shard.0); + return Err(Error::BadConfig); + } + } + Ok(config) } diff --git a/src/errors.rs b/src/errors.rs index 3dcbf74b1..1fc26bbf9 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -8,4 +8,5 @@ pub enum Error { // ServerTimeout, // DirtyServer, BadConfig, + AllServersDown, } diff --git a/src/pool.rs b/src/pool.rs index 57bc06623..49e13e5ce 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -113,7 +113,17 @@ impl ConnectionPool { None => 0, // TODO: pick a shard at random }; - loop { + let mut allowed_attempts = match role { + // Primary-specific queries get one attempt, if the primary is down, + // nothing we can do. + Some(Role::Primary) => 1, + + // Replicas get to try as many times as there are replicas. + Some(Role::Replica) => self.databases[shard].len(), + None => self.databases[shard].len(), + }; + + while allowed_attempts > 0 { // TODO: think about making this local, so multiple clients // don't compete for the same round-robin integer. // Especially since we're going to be skipping (see role selection below). @@ -121,21 +131,27 @@ impl ConnectionPool { self.round_robin.fetch_add(1, Ordering::SeqCst) % self.databases[shard].len(); let address = self.addresses[shard][index].clone(); - if self.is_banned(&address, shard) { - continue; - } - // Make sure you're getting a primary or a replica // as per request. match role { Some(role) => { - if address.role != role { + // If the client wants a specific role, + // we'll do our best to pick it, but if we only + // have one server in the cluster, it's probably only a primary + // (or only a replica), so the client will just get what we have. + if address.role != role && self.addresses[shard].len() > 1 { continue; } } None => (), }; + if self.is_banned(&address, shard, role) { + continue; + } + + allowed_attempts -= 1; + // Check if we can connect // TODO: implement query wait timeout, i.e. time to get a conn from the pool let mut conn = match self.databases[shard][index].get().await { @@ -183,6 +199,8 @@ impl ConnectionPool { } } } + + return Err(Error::AllServersDown); } /// Ban an address (i.e. replica). It no longer will serve @@ -204,7 +222,14 @@ impl ConnectionPool { /// Check if a replica can serve traffic. If all replicas are banned, /// we unban all of them. Better to try then not to. - pub fn is_banned(&self, address: &Address, shard: usize) -> bool { + pub fn is_banned(&self, address: &Address, shard: usize, role: Option) -> bool { + // If primary is requested explicitely, it can never be banned. + if Some(Role::Primary) == role { + return false; + } + + // If you're not asking for the primary, + // all databases are treated as replicas. let mut guard = self.banlist.lock().unwrap(); // Everything is banned = nothing is banned. diff --git a/tests/sharding/query_routing.sh b/tests/sharding/query_routing.sh index d1b2b845a..78aaa60e1 100644 --- a/tests/sharding/query_routing.sh +++ b/tests/sharding/query_routing.sh @@ -1,7 +1,12 @@ #/bin/bash +set -e # Setup all the shards. -sudo service postgresql restart +# sudo service postgresql restart + +echo "Giving Postgres 5 seconds to start up..." + +# sleep 5 psql -f query_routing_setup.sql @@ -9,4 +14,6 @@ psql -h 127.0.0.1 -p 6432 -f query_routing_test_insert.sql psql -h 127.0.0.1 -p 6432 -f query_routing_test_select.sql +psql -e -h 127.0.0.1 -p 6432 -f query_routing_test_primary_replica.sql + psql -f query_routing_test_validate.sql \ No newline at end of file diff --git a/tests/sharding/query_routing_test_primary_replica.sql b/tests/sharding/query_routing_test_primary_replica.sql new file mode 100644 index 000000000..06a734c42 --- /dev/null +++ b/tests/sharding/query_routing_test_primary_replica.sql @@ -0,0 +1,13 @@ +SET SERVER ROLE TO 'primary'; +SELECT 1; + +SET SERVER ROLE TO 'replica'; +SELECT 1; + +SET SHARDING KEY TO '1234'; +SET SERVER ROLE TO 'primary'; +SELECT 1; + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '4321'; +SELECT 1; \ No newline at end of file From 7be80940241f32aa919fc081d5f0183f8fc41975 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 9 Feb 2022 21:20:20 -0800 Subject: [PATCH 050/551] nl --- tests/sharding/query_routing.sh | 2 +- tests/sharding/query_routing_test_primary_replica.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sharding/query_routing.sh b/tests/sharding/query_routing.sh index 78aaa60e1..acc8532c0 100644 --- a/tests/sharding/query_routing.sh +++ b/tests/sharding/query_routing.sh @@ -16,4 +16,4 @@ psql -h 127.0.0.1 -p 6432 -f query_routing_test_select.sql psql -e -h 127.0.0.1 -p 6432 -f query_routing_test_primary_replica.sql -psql -f query_routing_test_validate.sql \ No newline at end of file +psql -f query_routing_test_validate.sql diff --git a/tests/sharding/query_routing_test_primary_replica.sql b/tests/sharding/query_routing_test_primary_replica.sql index 06a734c42..0188dbe45 100644 --- a/tests/sharding/query_routing_test_primary_replica.sql +++ b/tests/sharding/query_routing_test_primary_replica.sql @@ -10,4 +10,4 @@ SELECT 1; SET SERVER ROLE TO 'replica'; SET SHARDING KEY TO '4321'; -SELECT 1; \ No newline at end of file +SELECT 1; From e22f238b8cdd95c7b4bb22ef79a9c14e1c8f93d2 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 9 Feb 2022 21:25:17 -0800 Subject: [PATCH 051/551] readme --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 25e213d8e..d91ddd6a6 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,9 @@ See [sharding README](./tests/sharding/README.md) for sharding logic testing. 3. `COPY` protocol support. 4. Query cancellation. 5. Round-robin load balancing of replicas. -6. Banlist & failover +6. Banlist & failover. 7. Sharding! +8. Explicit query routing to primary or replicas. ### Session mode Each client owns its own server for the duration of the session. Commands like `SET` are allowed. @@ -56,7 +57,8 @@ this might be relevant given than this is a transactional pooler but if you're n ### Round-robin load balancing This is the novel part. PgBouncer doesn't support it and suggests we use DNS or a TCP proxy instead. We prefer to have everything as part of one package; arguably, it's easier to understand and optimize. -This pooler will round-robin between multiple replicas keeping load reasonably even. +This pooler will round-robin between multiple replicas keeping load reasonably even. If the primary is in +the pool as well, it'll be treated as a replica for read-only queries. ### Banlist & failover This is where it gets even more interesting. If we fail to connect to one of the replicas or it fails a health check, @@ -82,6 +84,19 @@ SET SHARDING KEY TO '1234'; This sharding key will be hashed and the pooler will select a shard to use for the next transaction. If the pooler is in session mode, this sharding key has to be set as the first query on startup & cannot be changed until the client re-connects. +### Explicit read/write query routing + +If you want to have the primary and replicas in the same pooler, you'd probably want to +route queries explicitely to the primary or replicas, depending if they are reads or writes (e.g `SELECT`s or `INSERT`/`UPDATE`, etc). To help with this, we introduce some more custom syntax: + +```sql +SET SERVER ROLE TO 'primary'; +SET SERVER ROLE TO 'replica'; +``` + +After executing this, the next transaction will be routed to the primary or replica respectively. By default, all queries will be load-balanced between all servers, so if the client wants to write or talk to the primary, they have to explicitely select it using the syntax above. + + ## Missing From 348261a20188bd3f6a0ba9acdf7498674a9fe774 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 08:35:25 -0800 Subject: [PATCH 052/551] more tests --- src/client.rs | 1 + .../query_routing_test_primary_replica.sql | 140 +++++++++++++++++- 2 files changed, 137 insertions(+), 4 deletions(-) diff --git a/src/client.rs b/src/client.rs index 7d7e27a21..d9bd074fa 100644 --- a/src/client.rs +++ b/src/client.rs @@ -180,6 +180,7 @@ impl Client { // - if in transaction mode, this lives for the duration of one transaction. let mut shard: Option = None; + // Active database role we want to talk to, e.g. primary or replica. let mut role: Option = None; loop { diff --git a/tests/sharding/query_routing_test_primary_replica.sql b/tests/sharding/query_routing_test_primary_replica.sql index 0188dbe45..358b073c6 100644 --- a/tests/sharding/query_routing_test_primary_replica.sql +++ b/tests/sharding/query_routing_test_primary_replica.sql @@ -1,13 +1,145 @@ SET SERVER ROLE TO 'primary'; -SELECT 1; +SET SHARDING KEY TO '1'; +INSERT INTO data (id, value) VALUES (1, 'value_1'); SET SERVER ROLE TO 'replica'; -SELECT 1; +SET SHARDING KEY TO '1'; +SELECT * FROM data WHERE id = 1; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '2'; +INSERT INTO data (id, value) VALUES (2, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '2'; +SELECT * FROM data WHERE id = 2; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '3'; +INSERT INTO data (id, value) VALUES (3, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '3'; +SELECT * FROM data WHERE id = 3; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '4'; +INSERT INTO data (id, value) VALUES (4, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '4'; +SELECT * FROM data WHERE id = 4; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '5'; +INSERT INTO data (id, value) VALUES (5, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '5'; +SELECT * FROM data WHERE id = 5; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '6'; +INSERT INTO data (id, value) VALUES (6, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '6'; +SELECT * FROM data WHERE id = 6; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '7'; +INSERT INTO data (id, value) VALUES (7, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '7'; +SELECT * FROM data WHERE id = 7; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '8'; +INSERT INTO data (id, value) VALUES (8, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '8'; +SELECT * FROM data WHERE id = 8; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '9'; +INSERT INTO data (id, value) VALUES (9, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '9'; +SELECT * FROM data WHERE id = 9; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '10'; +INSERT INTO data (id, value) VALUES (10, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '10'; +SELECT * FROM data WHERE id = 10; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '11'; +INSERT INTO data (id, value) VALUES (11, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '11'; +SELECT * FROM data WHERE id = 11; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '12'; +INSERT INTO data (id, value) VALUES (12, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '12'; +SELECT * FROM data WHERE id = 12; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '13'; +INSERT INTO data (id, value) VALUES (13, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '13'; +SELECT * FROM data WHERE id = 13; + +--- + +SET SERVER ROLE TO 'primary'; +SET SHARDING KEY TO '14'; +INSERT INTO data (id, value) VALUES (14, 'value_1'); + +SET SERVER ROLE TO 'replica'; +SET SHARDING KEY TO '14'; +SELECT * FROM data WHERE id = 14; + +--- -SET SHARDING KEY TO '1234'; SET SERVER ROLE TO 'primary'; SELECT 1; SET SERVER ROLE TO 'replica'; -SET SHARDING KEY TO '4321'; SELECT 1; From 269fc8582fc7b2cf76a67e7dc3a8e4f062bc7cec Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 08:54:06 -0800 Subject: [PATCH 053/551] pool fixes --- src/main.rs | 1 + src/pool.rs | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 9b084ecbe..3673e0c3c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,6 +73,7 @@ async fn main() { "> Healthcheck timeout: {}ms", config.general.healthcheck_timeout ); + println!("> Connection timeout: {}ms", config.general.connect_timeout); let pool = ConnectionPool::from_config(config.clone(), client_server_map.clone()).await; let transaction_mode = config.general.pool_mode == "transaction"; diff --git a/src/pool.rs b/src/pool.rs index 49e13e5ce..8cf3ada05 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -26,6 +26,7 @@ pub struct ConnectionPool { banlist: BanList, healthcheck_timeout: u64, ban_time: i64, + pool_size: u32, } impl ConnectionPool { @@ -96,6 +97,7 @@ impl ConnectionPool { banlist: Arc::new(Mutex::new(banlist)), healthcheck_timeout: config.general.healthcheck_timeout, ban_time: config.general.ban_time, + pool_size: config.general.pool_size, } } @@ -115,12 +117,29 @@ impl ConnectionPool { let mut allowed_attempts = match role { // Primary-specific queries get one attempt, if the primary is down, - // nothing we can do. - Some(Role::Primary) => 1, + // nothing we should do about it I think. It's dangerous to retry + // write queries. + Some(Role::Primary) => { + // Make sure we have a primary in the pool configured. + let primary_present = self.addresses[shard] + .iter() + .filter(|&db| db.role == Role::Primary) + .count(); + + // TODO: return this error to the client, so people don't have to look in + // the logs to figure out what happened. + if primary_present == 0 { + println!(">> Error: Primary requested but none are configured."); + return Err(Error::AllServersDown); + } + + // Primary gets one attempt. + 1 + } - // Replicas get to try as many times as there are replicas. - Some(Role::Replica) => self.databases[shard].len(), - None => self.databases[shard].len(), + // Replicas get to try as many times as there are replicas + // and connections in the pool. + _ => self.databases[shard].len() * self.pool_size as usize, }; while allowed_attempts > 0 { @@ -184,6 +203,9 @@ impl ConnectionPool { ">> Banning replica {} because of failed health check", index ); + // Don't leave a bad connection in the pool. + server.mark_bad(); + self.ban(&address, shard); continue; } @@ -194,6 +216,9 @@ impl ConnectionPool { ">> Banning replica {} because of health check timeout", index ); + // Don't leave a bad connection in the pool. + server.mark_bad(); + self.ban(&address, shard); continue; } From aec5a8e626b088ef4f797f195bc9cc6e9977ce4a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 09:07:10 -0800 Subject: [PATCH 054/551] config tests --- src/config.rs | 30 ++++++++++++++++++++++++++++-- tests/sharding/query_routing.sh | 2 +- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/config.rs b/src/config.rs index fe9206d84..b517c7890 100644 --- a/src/config.rs +++ b/src/config.rs @@ -77,13 +77,39 @@ pub async fn parse(path: &str) -> Result { } }; - // We use addresses as unique identifiers, - // let's make sure they are unique in the config as well. + // Quick config sanity check. for shard in &config.shards { + // We use addresses as unique identifiers, + // let's make sure they are unique in the config as well. let mut dup_check = HashSet::new(); + let mut primary_count = 0; for server in &shard.1.servers { dup_check.insert(server); + + // Check that we define only zero or one primary. + match server.2.as_ref() { + "primary" => primary_count += 1, + _ => (), + }; + + // Check role spelling. + match server.2.as_ref() { + "primary" => (), + "replica" => (), + _ => { + println!( + "> Shard {} server role must be either 'primary' or 'replica', got: '{}'", + shard.0, server.2 + ); + return Err(Error::BadConfig); + } + }; + } + + if primary_count > 1 { + println!("> Shard {} has more than on primary configured.", &shard.0); + return Err(Error::BadConfig); } if dup_check.len() != shard.1.servers.len() { diff --git a/tests/sharding/query_routing.sh b/tests/sharding/query_routing.sh index acc8532c0..d6098fa4d 100644 --- a/tests/sharding/query_routing.sh +++ b/tests/sharding/query_routing.sh @@ -8,7 +8,7 @@ echo "Giving Postgres 5 seconds to start up..." # sleep 5 -psql -f query_routing_setup.sql +# psql -f query_routing_setup.sql psql -h 127.0.0.1 -p 6432 -f query_routing_test_insert.sql From 5813c62a2e46497ec9390fb6610e603457b04247 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 10:37:49 -0800 Subject: [PATCH 055/551] removed atomic round-robin --- src/client.rs | 2 +- src/pool.rs | 75 +++++++++++++++++++++++++++++---------------------- 2 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/client.rs b/src/client.rs index d9bd074fa..b39769238 100644 --- a/src/client.rs +++ b/src/client.rs @@ -153,7 +153,7 @@ impl Client { } /// Client loop. We handle all messages between the client and the database here. - pub async fn handle(&mut self, pool: ConnectionPool) -> Result<(), Error> { + pub async fn handle(&mut self, mut pool: ConnectionPool) -> Result<(), Error> { // Special: cancelling existing running query if self.cancel_mode { let (process_id, secret_key, address, port) = { diff --git a/src/pool.rs b/src/pool.rs index 8cf3ada05..0b9b62be8 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -9,20 +9,20 @@ use crate::server::Server; use std::collections::HashMap; use std::sync::{ - atomic::{AtomicUsize, Ordering}, + // atomic::{AtomicUsize, Ordering}, Arc, Mutex, }; // Banlist: bad servers go in here. pub type BanList = Arc>>>; -pub type Counter = Arc; +// pub type Counter = Arc; pub type ClientServerMap = Arc>>; #[derive(Clone, Debug)] pub struct ConnectionPool { databases: Vec>>, addresses: Vec>, - round_robin: Counter, + round_robin: usize, banlist: BanList, healthcheck_timeout: u64, ban_time: i64, @@ -90,10 +90,13 @@ impl ConnectionPool { banlist.push(HashMap::new()); } + assert_eq!(shards.len(), addresses.len()); + let address_len = addresses.len(); + ConnectionPool { databases: shards, addresses: addresses, - round_robin: Arc::new(AtomicUsize::new(0)), + round_robin: rand::random::() % address_len, // Start at a random replica banlist: Arc::new(Mutex::new(banlist)), healthcheck_timeout: config.general.healthcheck_timeout, ban_time: config.general.ban_time, @@ -103,7 +106,7 @@ impl ConnectionPool { /// Get a connection from the pool. pub async fn get( - &self, + &mut self, shard: Option, role: Option, ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { @@ -115,40 +118,48 @@ impl ConnectionPool { None => 0, // TODO: pick a shard at random }; - let mut allowed_attempts = match role { - // Primary-specific queries get one attempt, if the primary is down, - // nothing we should do about it I think. It's dangerous to retry - // write queries. - Some(Role::Primary) => { - // Make sure we have a primary in the pool configured. - let primary_present = self.addresses[shard] + let addresses = &self.addresses[shard]; + + // Make sure if a specific role is requested, it's available in the pool. + match role { + Some(role) => { + let role_count = addresses .iter() .filter(|&db| db.role == Role::Primary) .count(); - // TODO: return this error to the client, so people don't have to look in - // the logs to figure out what happened. - if primary_present == 0 { - println!(">> Error: Primary requested but none are configured."); + if role_count == 0 { + println!( + ">> Error: Role '{:?}' requested, but none are configured.", + role + ); + return Err(Error::AllServersDown); } - - // Primary gets one attempt. - 1 } + // Any role should be present. + _ => (), + }; + + let mut allowed_attempts = match role { + // Primary-specific queries get one attempt, if the primary is down, + // nothing we should do about it I think. It's dangerous to retry + // write queries. + Some(Role::Primary) => 1, + // Replicas get to try as many times as there are replicas // and connections in the pool. _ => self.databases[shard].len() * self.pool_size as usize, }; while allowed_attempts > 0 { - // TODO: think about making this local, so multiple clients - // don't compete for the same round-robin integer. - // Especially since we're going to be skipping (see role selection below). - let index = - self.round_robin.fetch_add(1, Ordering::SeqCst) % self.databases[shard].len(); - let address = self.addresses[shard][index].clone(); + // Round-robin each client's queries. + // If a client only sends one query and then disconnects, it doesn't matter + // which replica it'll go to. + self.round_robin += 1; + let index = self.round_robin % addresses.len(); + let address = &addresses[index]; // Make sure you're getting a primary or a replica // as per request. @@ -158,14 +169,14 @@ impl ConnectionPool { // we'll do our best to pick it, but if we only // have one server in the cluster, it's probably only a primary // (or only a replica), so the client will just get what we have. - if address.role != role && self.addresses[shard].len() > 1 { + if address.role != role && addresses.len() > 1 { continue; } } None => (), }; - if self.is_banned(&address, shard, role) { + if self.is_banned(address, shard, role) { continue; } @@ -177,13 +188,13 @@ impl ConnectionPool { Ok(conn) => conn, Err(err) => { println!(">> Banning replica {}, error: {:?}", index, err); - self.ban(&address, shard); + self.ban(address, shard); continue; } }; if !with_health_check { - return Ok((conn, address)); + return Ok((conn, address.clone())); } // // Check if this server is alive with a health check @@ -197,7 +208,7 @@ impl ConnectionPool { { // Check if health check succeeded Ok(res) => match res { - Ok(_) => return Ok((conn, address)), + Ok(_) => return Ok((conn, address.clone())), Err(_) => { println!( ">> Banning replica {} because of failed health check", @@ -206,7 +217,7 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(&address, shard); + self.ban(address, shard); continue; } }, @@ -219,7 +230,7 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(&address, shard); + self.ban(address, shard); continue; } } From fbdb09b6a92b8327f13dc544106ae6ce299b0b74 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 10:37:55 -0800 Subject: [PATCH 056/551] todo complete --- src/pool.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pool.rs b/src/pool.rs index 0b9b62be8..9905e3605 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -183,7 +183,6 @@ impl ConnectionPool { allowed_attempts -= 1; // Check if we can connect - // TODO: implement query wait timeout, i.e. time to get a conn from the pool let mut conn = match self.databases[shard][index].get().await { Ok(conn) => conn, Err(err) => { From 64b0481d5d8ff069ea271a600f4e9659cfd2c421 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 10:38:06 -0800 Subject: [PATCH 057/551] fmt --- src/pool.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pool.rs b/src/pool.rs index 9905e3605..cbe4d1adf 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -10,7 +10,8 @@ use crate::server::Server; use std::collections::HashMap; use std::sync::{ // atomic::{AtomicUsize, Ordering}, - Arc, Mutex, + Arc, + Mutex, }; // Banlist: bad servers go in here. From 32c5aca3404fe81764f7b6489d73f24abeb60b24 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 10:50:19 -0800 Subject: [PATCH 058/551] test ci --- .circleci/config.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 35826ec7c..75fdc0937 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,20 +10,39 @@ jobs: # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - image: cimg/rust:1.58.1 + - image: cimg/postgres:14.0 + auth: + username: sharding_user + password: sharding_user + environment: + POSTGRES_USER: root + POSTGRES_DB: shard0 # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: - checkout - restore_cache: - key: cargo-lock-2-{{ checksum "Cargo.lock" }} + key: cargo-lock-3-{{ checksum "Cargo.lock" }} + - run: + name: "Install dependencies" + command: "apt-get update && apt-get install postgresql-contrib-12 postgresql-client-12" - run: name: "Build" - command: "cargo build" + command: "cargo build --release" - run: name: "Test" command: "cargo test" + - run: + name: "Launch in background" + command: "./target/release/pgcat &" + - run: + name: "PgBench initialize" + command: "pgbench -i -h 127.0.0.1 -p 6432" + - run: + name: "PgBench run" + command: "pgbench -h 127.0.0.1 -p 6432 --protocol extended -t 100 -c 2" - save_cache: - key: cargo-lock-2-{{ checksum "Cargo.lock" }} + key: cargo-lock-3-{{ checksum "Cargo.lock" }} paths: - target - ~/.cargo From a0bed1e0b333644c0af7eb5827255b6893183fde Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 10:53:15 -0800 Subject: [PATCH 059/551] hmm --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 75fdc0937..df9968d41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,10 +12,10 @@ jobs: - image: cimg/rust:1.58.1 - image: cimg/postgres:14.0 auth: - username: sharding_user - password: sharding_user + username: mydockerhub-user + password: $DOCKERHUB_PASSWORD environment: - POSTGRES_USER: root + POSTGRES_USER: sharding_user POSTGRES_DB: shard0 # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps From d92539ddae1af1a21e74da8ac5d9ff081d70ac21 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 10:54:06 -0800 Subject: [PATCH 060/551] sudo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index df9968d41..5de73fedf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: key: cargo-lock-3-{{ checksum "Cargo.lock" }} - run: name: "Install dependencies" - command: "apt-get update && apt-get install postgresql-contrib-12 postgresql-client-12" + command: "sudo apt-get update && sudo apt-get install postgresql-contrib-12 postgresql-client-12" - run: name: "Build" command: "cargo build --release" From 27f3bd3c0dd8af9976c2bc241a732e2db7a41556 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 10:59:45 -0800 Subject: [PATCH 061/551] background --- .circleci/config.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5de73fedf..e66f1d781 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: key: cargo-lock-3-{{ checksum "Cargo.lock" }} - run: name: "Install dependencies" - command: "sudo apt-get update && sudo apt-get install postgresql-contrib-12 postgresql-client-12" + command: "sudo apt-get update && sudo apt-get install -y postgresql-contrib-12 postgresql-client-12" - run: name: "Build" command: "cargo build --release" @@ -34,13 +34,7 @@ jobs: command: "cargo test" - run: name: "Launch in background" - command: "./target/release/pgcat &" - - run: - name: "PgBench initialize" - command: "pgbench -i -h 127.0.0.1 -p 6432" - - run: - name: "PgBench run" - command: "pgbench -h 127.0.0.1 -p 6432 --protocol extended -t 100 -c 2" + command: "./target/release/pgcat & pgbench -i -h 127.0.0.1 -p 6432 & pgbench -h 127.0.0.1 -p 6432 --protocol extended -t 100 -c 2" - save_cache: key: cargo-lock-3-{{ checksum "Cargo.lock" }} paths: From b0ddce6b4f6846e08fc43a4464c5f3d7f888e1d5 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 11:08:57 -0800 Subject: [PATCH 062/551] tests --- .circleci/config.yml | 14 +++++++------- .circleci/run_tests.sh | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 .circleci/run_tests.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index e66f1d781..bd5b396bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,28 +15,28 @@ jobs: username: mydockerhub-user password: $DOCKERHUB_PASSWORD environment: - POSTGRES_USER: sharding_user - POSTGRES_DB: shard0 + POSTGRES_USER: postgres + POSTGRES_DB: postgres # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: - checkout - restore_cache: - key: cargo-lock-3-{{ checksum "Cargo.lock" }} + key: cargo-lock-2-{{ checksum "Cargo.lock" }} - run: name: "Install dependencies" command: "sudo apt-get update && sudo apt-get install -y postgresql-contrib-12 postgresql-client-12" - run: name: "Build" - command: "cargo build --release" + command: "cargo build" - run: name: "Test" command: "cargo test" - run: - name: "Launch in background" - command: "./target/release/pgcat & pgbench -i -h 127.0.0.1 -p 6432 & pgbench -h 127.0.0.1 -p 6432 --protocol extended -t 100 -c 2" + name: "Test end-to-end" + command: "bash .circleci/run_tests.sh" - save_cache: - key: cargo-lock-3-{{ checksum "Cargo.lock" }} + key: cargo-lock-2-{{ checksum "Cargo.lock" }} paths: - target - ~/.cargo diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh new file mode 100644 index 000000000..fe2763493 --- /dev/null +++ b/.circleci/run_tests.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +./target/debug/pgcat & + +sleep 1 + +psql -f tests/sharding/query_routing_setup.sql + +# Setup PgBench +pgbench -i -h 127.0.0.1 -p 6432 + +# Run it +pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 + +psql -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_insert.sql + +psql -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql + +# psql -f tests/sharding/query_routing_test_validate.sql + +psql -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql \ No newline at end of file From 6c82e9de11e21df37227d7a901d57f2e819b48a6 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 11:11:56 -0800 Subject: [PATCH 063/551] print whats going on --- .circleci/run_tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index fe2763493..2385644c0 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -1,12 +1,13 @@ #!/bin/bash set -e +set -o xtrace ./target/debug/pgcat & sleep 1 -psql -f tests/sharding/query_routing_setup.sql +psql -h 127.0.0.1 -p 5432 -f tests/sharding/query_routing_setup.sql # Setup PgBench pgbench -i -h 127.0.0.1 -p 6432 From 94b3efd38fb77e783e228efae4e3673039ffce38 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 11:13:31 -0800 Subject: [PATCH 064/551] hmm --- .circleci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 2385644c0..2b752ab2d 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -7,7 +7,7 @@ set -o xtrace sleep 1 -psql -h 127.0.0.1 -p 5432 -f tests/sharding/query_routing_setup.sql +psql -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql # Setup PgBench pgbench -i -h 127.0.0.1 -p 6432 From 4f9110365f2a1da586cc4608f0814148a240b4b3 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 11:16:08 -0800 Subject: [PATCH 065/551] print whats going on --- .circleci/run_tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 2b752ab2d..1150862d5 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -7,7 +7,7 @@ set -o xtrace sleep 1 -psql -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql +psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql # Setup PgBench pgbench -i -h 127.0.0.1 -p 6432 @@ -15,10 +15,10 @@ pgbench -i -h 127.0.0.1 -p 6432 # Run it pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 -psql -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_insert.sql +psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_insert.sql -psql -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql +psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql # psql -f tests/sharding/query_routing_test_validate.sql -psql -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql \ No newline at end of file +psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql \ No newline at end of file From 498320f83b4a3356d05de954eddfc054475e539a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 11:20:33 -0800 Subject: [PATCH 066/551] maybe were breaking the terminal? --- .circleci/run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 1150862d5..4307e42df 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -17,8 +17,8 @@ pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_insert.sql -psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql +psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql > /dev/null # psql -f tests/sharding/query_routing_test_validate.sql -psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql \ No newline at end of file +psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null \ No newline at end of file From 1f8f56ea9f7820caf03e89520d84c15e7e4361a4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 13:48:56 -0800 Subject: [PATCH 067/551] max_workers = 4 = so much faster --- Cargo.toml | 2 +- src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fd66048c9..70e0f9146 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,4 +17,4 @@ sha-1 = "0.10" toml = "0.5" serde = "1" serde_derive = "1" -regex = "1" +regex = "1" \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 3673e0c3c..aed7cae76 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,7 +40,7 @@ mod sharding; use pool::{ClientServerMap, ConnectionPool}; /// Main! -#[tokio::main] +#[tokio::main(worker_threads = 4)] async fn main() { println!("> Welcome to PgCat! Meow."); From f5dc7eadfaa7152abf7116c8f00abff849d471c0 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 14:02:24 -0800 Subject: [PATCH 068/551] benchmarks --- README.md | 259 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 185 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index d91ddd6a6..ffee1001c 100644 --- a/README.md +++ b/README.md @@ -110,114 +110,225 @@ You can setup PgBench locally through PgCat: pgbench -h 127.0.0.1 -p 6432 -i ``` -Coincidenly, this uses `COPY` so you can test if that works. +Coincidenly, this uses `COPY` so you can test if that works. Additionally, we'll be running the following PgBench configurations: + +1. 16 clients, 2 threads +2. 32 clients, 2 threads +3. 64 clients, 2 threads +4. 128 clients, 2 threads + +All queries will be `SELECT` only (`-S`) just so disks don't get in the way, since the dataset will be effectively all in RAM. ### PgBouncer +#### Config + +```ini +[databases] +shard0 = host=localhost port=5432 user=sharding_user password=sharding_user + +pool_mode = transaction +max_client_conn = 1000 +``` + +#### Runs + + ``` -$ pgbench -i -h 127.0.0.1 -p 6432 && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended -dropping old tables... -creating tables... -generating data... -100000 of 100000 tuples (100%) done (elapsed 0.01 s, remaining 0.00 s) -vacuuming... -creating primary keys... -done. +$ pgbench -t 1000 -c 16 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 + starting vacuum...end. -transaction type: +transaction type: scaling factor: 1 -query mode: simple -number of clients: 1 -number of threads: 1 +query mode: extended +number of clients: 16 +number of threads: 2 number of transactions per client: 1000 -number of transactions actually processed: 1000/1000 -latency average = 1.089 ms -tps = 918.687098 (including connections establishing) -tps = 918.847790 (excluding connections establishing) +number of transactions actually processed: 16000/16000 +latency average = 0.155 ms +tps = 103417.377469 (including connections establishing) +tps = 103510.639935 (excluding connections establishing) + + +$ pgbench -t 1000 -c 32 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 + starting vacuum...end. -transaction type: +transaction type: scaling factor: 1 query mode: extended -number of clients: 1 -number of threads: 1 +number of clients: 32 +number of threads: 2 number of transactions per client: 1000 -number of transactions actually processed: 1000/1000 -latency average = 1.136 ms -tps = 880.622009 (including connections establishing) -tps = 880.769550 (excluding connections establishing) +number of transactions actually processed: 32000/32000 +latency average = 0.290 ms +tps = 110325.939785 (including connections establishing) +tps = 110386.513435 (excluding connections establishing) + + +$ pgbench -t 1000 -c 64 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 + +starting vacuum...end. +transaction type: +scaling factor: 1 +query mode: extended +number of clients: 64 +number of threads: 2 +number of transactions per client: 1000 +number of transactions actually processed: 64000/64000 +latency average = 0.692 ms +tps = 92470.427412 (including connections establishing) +tps = 92618.389350 (excluding connections establishing) + +$ pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 + +starting vacuum...end. +transaction type: +scaling factor: 1 +query mode: extended +number of clients: 128 +number of threads: 2 +number of transactions per client: 1000 +number of transactions actually processed: 128000/128000 +latency average = 1.406 ms +tps = 91013.429985 (including connections establishing) +tps = 91067.583928 (excluding connections establishing) ``` ### PgCat +#### Config + +The only thing that matters here is the number of workers in the Tokio pool. Make sure to set it to < than the number of your CPU cores. +Also account for hyper-threading, so if you have that, take the number you got above and divide it by two, that way only "real" cores serving +requests. + +My setup is 16 threads, 8 cores (`htop` shows as 16 CPUs), so I set the `max_workers` in Tokio to 4. Too many, and it starts conflicting with PgBench +which is also running on the same system. + +#### Runs + ``` -$ pgbench -i -h 127.0.0.1 -p 6432 && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol simple && pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended -dropping old tables... -creating tables... -generating data... -100000 of 100000 tuples (100%) done (elapsed 0.01 s, remaining 0.00 s) -vacuuming... -creating primary keys... -done. +$ pgbench -t 1000 -c 16 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended starting vacuum...end. -transaction type: +transaction type: scaling factor: 1 -query mode: simple -number of clients: 1 -number of threads: 1 +query mode: extended +number of clients: 16 +number of threads: 2 number of transactions per client: 1000 -number of transactions actually processed: 1000/1000 -latency average = 1.142 ms -tps = 875.645437 (including connections establishing) -tps = 875.799995 (excluding connections establishing) +number of transactions actually processed: 16000/16000 +latency average = 0.164 ms +tps = 97705.088232 (including connections establishing) +tps = 97872.216045 (excluding connections establishing) + + +$ pgbench -t 1000 -c 32 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended + +starting vacuum...end. +transaction type: +scaling factor: 1 +query mode: extended +number of clients: 32 +number of threads: 2 +number of transactions per client: 1000 +number of transactions actually processed: 32000/32000 +latency average = 0.288 ms +tps = 111300.488119 (including connections establishing) +tps = 111413.107800 (excluding connections establishing) + + +$ pgbench -t 1000 -c 64 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended + +starting vacuum...end. +transaction type: +scaling factor: 1 +query mode: extended +number of clients: 64 +number of threads: 2 +number of transactions per client: 1000 +number of transactions actually processed: 64000/64000 +latency average = 0.556 ms +tps = 115190.496139 (including connections establishing) +tps = 115247.521295 (excluding connections establishing) + +$ pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended + starting vacuum...end. -transaction type: +transaction type: scaling factor: 1 query mode: extended -number of clients: 1 -number of threads: 1 +number of clients: 128 +number of threads: 2 number of transactions per client: 1000 -number of transactions actually processed: 1000/1000 -latency average = 1.181 ms -tps = 846.539176 (including connections establishing) -tps = 846.713636 (excluding connections establishing) +number of transactions actually processed: 128000/128000 +latency average = 1.135 ms +tps = 112770.562239 (including connections establishing) +tps = 112796.502381 (excluding connections establishing) ``` ### Direct Postgres +Always good to have a base line. Note, at 128 clients, having our pooler ends up being faster. + +#### Runs + ``` -$ pgbench -i -h 127.0.0.1 -p 5432 && pgbench -t 1000 -p 5432 -h 127.0.0.1 --protocol simple && pgbench -t 1000 -p -5432 -h 127.0.0.1 --protocol extended -Password: -dropping old tables... -creating tables... -generating data... -100000 of 100000 tuples (100%) done (elapsed 0.01 s, remaining 0.00 s) -vacuuming... -creating primary keys... -done. -Password: +$ pgbench -t 1000 -c 16 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 +Password: starting vacuum...end. -transaction type: +transaction type: scaling factor: 1 -query mode: simple -number of clients: 1 -number of threads: 1 +query mode: extended +number of clients: 16 +number of threads: 2 number of transactions per client: 1000 -number of transactions actually processed: 1000/1000 -latency average = 0.902 ms -tps = 1109.014867 (including connections establishing) -tps = 1112.318595 (excluding connections establishing) -Password: +number of transactions actually processed: 16000/16000 +latency average = 0.115 ms +tps = 139443.955722 (including connections establishing) +tps = 142314.859075 (excluding connections establishing) + +$ pgbench -t 1000 -c 32 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 +Password: +starting vacuum...end. +transaction type: +scaling factor: 1 +query mode: extended +number of clients: 32 +number of threads: 2 +number of transactions per client: 1000 +number of transactions actually processed: 32000/32000 +latency average = 0.212 ms +tps = 150644.840891 (including connections establishing) +tps = 152218.499430 (excluding connections establishing) + +$ pgbench -t 1000 -c 64 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 +Password: +starting vacuum...end. +transaction type: +scaling factor: 1 +query mode: extended +number of clients: 64 +number of threads: 2 +number of transactions per client: 1000 +number of transactions actually processed: 64000/64000 +latency average = 0.420 ms +tps = 152517.663404 (including connections establishing) +tps = 153319.188482 (excluding connections establishing) + +$ pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 +Password: starting vacuum...end. -transaction type: +pgbench: error: connection to database "shard0" failed: FATAL: sorry, too many clients already +FATAL: sorry, too many clients already +transaction type: scaling factor: 1 query mode: extended -number of clients: 1 -number of threads: 1 +number of clients: 128 +number of threads: 2 number of transactions per client: 1000 -number of transactions actually processed: 1000/1000 -latency average = 0.931 ms -tps = 1074.017747 (including connections establishing) -tps = 1077.121752 (excluding connections establishing) +number of transactions actually processed: 64000/128000 +latency average = 1.324 ms +tps = 96692.385260 (including connections establishing) +tps = 96854.431104 (excluding connections establishing) ``` From eafbba22a264ff0072f11cba2f8986bfe727892d Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 14:03:38 -0800 Subject: [PATCH 069/551] readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ffee1001c..8197e6056 100644 --- a/README.md +++ b/README.md @@ -127,10 +127,13 @@ All queries will be `SELECT` only (`-S`) just so disks don't get in the way, sin [databases] shard0 = host=localhost port=5432 user=sharding_user password=sharding_user +[pgbouncer] pool_mode = transaction max_client_conn = 1000 ``` +Everything else stays default. + #### Runs From b718d0fc55158f74edfa65d808e7d539d0ed850f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 14:13:47 -0800 Subject: [PATCH 070/551] bench setup --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 8197e6056..9b9461f08 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,11 @@ Coincidenly, this uses `COPY` so you can test if that works. Additionally, we'll All queries will be `SELECT` only (`-S`) just so disks don't get in the way, since the dataset will be effectively all in RAM. +My setup: + +- 8 cores, 16 hyperthreaded (AMD Ryzen 5800X) +- 32GB RAM (doesn't matter for this benchmark, except to prove that Postgres will fit the whole dataset into RAM) + ### PgBouncer #### Config From 7e058fc1000e25612c1e3a1f9d074366df01705a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 14:25:04 -0800 Subject: [PATCH 071/551] wrong benchmark --- README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9b9461f08..050b1992a 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ tps = 112796.502381 (excluding connections establishing) ### Direct Postgres -Always good to have a base line. Note, at 128 clients, having our pooler ends up being faster. +Always good to have a base line. #### Runs @@ -327,16 +327,14 @@ tps = 153319.188482 (excluding connections establishing) $ pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 Password: starting vacuum...end. -pgbench: error: connection to database "shard0" failed: FATAL: sorry, too many clients already -FATAL: sorry, too many clients already transaction type: scaling factor: 1 query mode: extended number of clients: 128 number of threads: 2 number of transactions per client: 1000 -number of transactions actually processed: 64000/128000 -latency average = 1.324 ms -tps = 96692.385260 (including connections establishing) -tps = 96854.431104 (excluding connections establishing) +number of transactions actually processed: 128000/128000 +latency average = 0.854 ms +tps = 149818.594087 (including connections establishing) +tps = 150200.603049 (excluding connections establishing) ``` From 80c7ef0156c80ba948a1191e8c6656366e23bfb4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Feb 2022 17:05:20 -0800 Subject: [PATCH 072/551] once_cell is way faster --- Cargo.lock | 2 ++ Cargo.toml | 4 +++- src/client.rs | 33 ++++++++++++++++----------------- src/main.rs | 10 ++++++++++ 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e6f79e17..1f18149f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -323,6 +323,8 @@ dependencies = [ "bytes", "chrono", "md-5", + "num_cpus", + "once_cell", "rand", "regex", "serde", diff --git a/Cargo.toml b/Cargo.toml index 70e0f9146..6db820ca3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,4 +17,6 @@ sha-1 = "0.10" toml = "0.5" serde = "1" serde_derive = "1" -regex = "1" \ No newline at end of file +regex = "1" +num_cpus = "1" +once_cell = "1" diff --git a/src/client.rs b/src/client.rs index b39769238..146030979 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,6 +2,7 @@ /// We are pretending to the server in this scenario, /// and this module implements that. use bytes::{Buf, BufMut, BytesMut}; +use once_cell::sync::OnceCell; use regex::Regex; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; @@ -14,8 +15,11 @@ use crate::pool::{ClientServerMap, ConnectionPool}; use crate::server::Server; use crate::sharding::Sharder; -const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+';"; -const ROLE_REGEX: &str = r"SET SERVER ROLE TO '(PRIMARY|REPLICA)';"; +pub const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+';"; +pub const ROLE_REGEX: &str = r"SET SERVER ROLE TO '(PRIMARY|REPLICA)';"; + +pub static SHARDING_REGEX_RE: OnceCell = OnceCell::new(); +pub static ROLE_REGEX_RE: OnceCell = OnceCell::new(); /// The client state. One of these is created per client. pub struct Client { @@ -44,12 +48,6 @@ pub struct Client { // Clients are mapped to servers while they use them. This allows a client // to connect and cancel a query. client_server_map: ClientServerMap, - - // sharding regex - sharding_regex: Regex, - - // role detection regex - role_regex: Regex, } impl Client { @@ -61,9 +59,6 @@ impl Client { client_server_map: ClientServerMap, transaction_mode: bool, ) -> Result { - let sharding_regex = Regex::new(SHARDING_REGEX).unwrap(); - let role_regex = Regex::new(ROLE_REGEX).unwrap(); - loop { // Could be StartupMessage or SSLRequest // which makes this variable length. @@ -119,8 +114,6 @@ impl Client { process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, - sharding_regex: sharding_regex, - role_regex: role_regex, }); } @@ -140,8 +133,6 @@ impl Client { process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, - sharding_regex: sharding_regex, - role_regex: role_regex, }); } @@ -414,8 +405,12 @@ impl Client { let len = buf.get_i32(); let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); // Don't read the ternminating null + let rgx = match SHARDING_REGEX_RE.get() { + Some(r) => r, + None => return None, + }; - if self.sharding_regex.is_match(&query) { + if rgx.is_match(&query) { let shard = query.split("'").collect::>()[1]; match shard.parse::() { Ok(shard) => { @@ -441,10 +436,14 @@ impl Client { let len = buf.get_i32(); let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); + let rgx = match ROLE_REGEX_RE.get() { + Some(r) => r, + None => return None, + }; // Copy / paste from above. If we get one more of these use cases, // it'll be time to abstract :). - if self.role_regex.is_match(&query) { + if rgx.is_match(&query) { let role = query.split("'").collect::>()[1]; match role { "PRIMARY" => Some(Role::Primary), diff --git a/src/main.rs b/src/main.rs index aed7cae76..8ba9cacde 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,11 +17,14 @@ extern crate async_trait; extern crate bb8; extern crate bytes; extern crate md5; +extern crate num_cpus; +extern crate once_cell; extern crate serde; extern crate serde_derive; extern crate tokio; extern crate toml; +use regex::Regex; use tokio::net::TcpListener; use std::collections::HashMap; @@ -44,6 +47,13 @@ use pool::{ClientServerMap, ConnectionPool}; async fn main() { println!("> Welcome to PgCat! Meow."); + client::SHARDING_REGEX_RE + .set(Regex::new(client::SHARDING_REGEX).unwrap()) + .unwrap(); + client::ROLE_REGEX_RE + .set(Regex::new(client::ROLE_REGEX).unwrap()) + .unwrap(); + let config = match config::parse("pgcat.toml").await { Ok(config) => config, Err(err) => { From 08939f3e9795a0ab0c355b2f0dbb259bfd4d13b9 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 11 Feb 2022 11:19:40 -0800 Subject: [PATCH 073/551] add default server role; bug fix --- pgcat.toml | 12 ++++++++++++ src/client.rs | 21 ++++++++++++++------- src/config.rs | 20 ++++++++++++++++++++ src/main.rs | 19 ++++++++++++++++++- src/pool.rs | 5 +---- 5 files changed, 65 insertions(+), 12 deletions(-) diff --git a/pgcat.toml b/pgcat.toml index ffcf7220d..44bcd00fd 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -69,3 +69,15 @@ servers = [ # [ "127.0.1.1", 5432, "replica" ], ] database = "shard2" + + +# Settings for our query routing layer. +[query_router] + +# If the client doesn't specify, route traffic to +# this role by default. +# +# any: round-robin between primary and replicas, +# replica: round-robin between replicas only without touching the primary, +# primary: all queries go to the primary unless otherwise specified. +default_role = "any" \ No newline at end of file diff --git a/src/client.rs b/src/client.rs index 146030979..a9e7a6076 100644 --- a/src/client.rs +++ b/src/client.rs @@ -48,6 +48,10 @@ pub struct Client { // Clients are mapped to servers while they use them. This allows a client // to connect and cancel a query. client_server_map: ClientServerMap, + + // Unless client specifies, route queries to the servers that have this role, + // e.g. primary or replicas or any. + default_server_role: Option, } impl Client { @@ -58,6 +62,7 @@ impl Client { mut stream: TcpStream, client_server_map: ClientServerMap, transaction_mode: bool, + default_server_role: Option, ) -> Result { loop { // Could be StartupMessage or SSLRequest @@ -114,6 +119,7 @@ impl Client { process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, + default_server_role: default_server_role, }); } @@ -133,6 +139,7 @@ impl Client { process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, + default_server_role: default_server_role, }); } @@ -172,7 +179,7 @@ impl Client { let mut shard: Option = None; // Active database role we want to talk to, e.g. primary or replica. - let mut role: Option = None; + let mut role: Option = self.default_server_role; loop { // Read a complete message from the client, which normally would be @@ -275,7 +282,7 @@ impl Client { // Release server if !server.in_transaction() && self.transaction_mode { shard = None; - role = None; + role = self.default_server_role; break; } } @@ -338,7 +345,7 @@ impl Client { // Release server if !server.in_transaction() && self.transaction_mode { shard = None; - role = None; + role = self.default_server_role; break; } } @@ -366,7 +373,7 @@ impl Client { if !server.in_transaction() && self.transaction_mode { println!("Releasing after copy done"); shard = None; - role = None; + role = self.default_server_role; break; } } @@ -382,7 +389,7 @@ impl Client { } /// Release the server from being mine. I can't cancel its queries anymore. - pub fn release(&mut self) { + pub fn release(&self) { let mut guard = self.client_server_map.lock().unwrap(); guard.remove(&(self.process_id, self.secret_key)); } @@ -390,7 +397,7 @@ impl Client { /// Determine if the query is part of our special syntax, extract /// the shard key, and return the shard to query based on Postgres' /// PARTITION BY HASH function. - fn select_shard(&mut self, mut buf: BytesMut, shards: usize) -> Option { + fn select_shard(&self, mut buf: BytesMut, shards: usize) -> Option { let code = buf.get_u8() as char; // Only supporting simpe protocol here, so @@ -425,7 +432,7 @@ impl Client { } // Pick a primary or a replica from the pool. - fn select_role(&mut self, mut buf: BytesMut) -> Option { + fn select_role(&self, mut buf: BytesMut) -> Option { let code = buf.get_u8() as char; // Same story as select_shard() above. diff --git a/src/config.rs b/src/config.rs index b517c7890..79c2c0b3a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -43,11 +43,17 @@ pub struct Shard { pub database: String, } +#[derive(Deserialize, Debug, Clone)] +pub struct QueryRouter { + pub default_role: String, +} + #[derive(Deserialize, Debug, Clone)] pub struct Config { pub general: General, pub user: User, pub shards: HashMap, + pub query_router: QueryRouter, } /// Parse the config. @@ -118,6 +124,19 @@ pub async fn parse(path: &str) -> Result { } } + match config.query_router.default_role.as_ref() { + "any" => (), + "primary" => (), + "replica" => (), + other => { + println!( + "> Query router default_role must be 'primary', 'replica', or 'any', got: '{}'", + other + ); + return Err(Error::BadConfig); + } + }; + Ok(config) } @@ -132,5 +151,6 @@ mod test { assert_eq!(config.shards.len(), 3); assert_eq!(config.shards["1"].servers[0].0, "127.0.0.1"); assert_eq!(config.shards["0"].servers[0].2, "primary"); + assert_eq!(config.query_router.default_role, "any"); } } diff --git a/src/main.rs b/src/main.rs index 8ba9cacde..05903d661 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,6 +40,7 @@ mod sharding; // Support for query cancellation: this maps our process_ids and // secret keys to the backend's. +use config::Role; use pool::{ClientServerMap, ConnectionPool}; /// Main! @@ -87,6 +88,15 @@ async fn main() { let pool = ConnectionPool::from_config(config.clone(), client_server_map.clone()).await; let transaction_mode = config.general.pool_mode == "transaction"; + let default_server_role = match config.query_router.default_role.as_ref() { + "any" => None, + "primary" => Some(Role::Primary), + "replica" => Some(Role::Replica), + _ => { + println!("> Config error, got unexpected query_router.default_role."); + return; + } + }; println!("> Waiting for clients..."); @@ -109,7 +119,14 @@ async fn main() { addr, transaction_mode ); - match client::Client::startup(socket, client_server_map, transaction_mode).await { + match client::Client::startup( + socket, + client_server_map, + transaction_mode, + default_server_role, + ) + .await + { Ok(mut client) => { println!(">> Client {:?} authenticated successfully!", addr); diff --git a/src/pool.rs b/src/pool.rs index cbe4d1adf..8c92bc6d9 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -124,10 +124,7 @@ impl ConnectionPool { // Make sure if a specific role is requested, it's available in the pool. match role { Some(role) => { - let role_count = addresses - .iter() - .filter(|&db| db.role == Role::Primary) - .count(); + let role_count = addresses.iter().filter(|&db| db.role == role).count(); if role_count == 0 { println!( From 3339844a5e7644ff54f97a4a2a6e80777e3b44a4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 11 Feb 2022 11:21:32 -0800 Subject: [PATCH 074/551] nl --- pgcat.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgcat.toml b/pgcat.toml index 44bcd00fd..db5c3f2f5 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -80,4 +80,4 @@ database = "shard2" # any: round-robin between primary and replicas, # replica: round-robin between replicas only without touching the primary, # primary: all queries go to the primary unless otherwise specified. -default_role = "any" \ No newline at end of file +default_role = "any" From 7f88148558619ef936857ab171e1cced59a38e34 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 11 Feb 2022 12:02:08 -0800 Subject: [PATCH 075/551] docker image (#9) * docker image * nl --- .dockerignore | 4 ++++ Dockerfile | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..ff42aaaaf --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +target/ +tests/ +tracing/ +.circleci/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..37706259d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM rust:1.58-slim-buster AS builder +COPY . /app +WORKDIR /app +RUN cargo build --release + +FROM debian:buster-slim +COPY --from=builder /app/target/release/pgcat /usr/bin/pgcat +COPY --from=builder /app/pgcat.toml /etc/pgcat/pgcat.toml +WORKDIR /etc/pgcat +ENTRYPOINT ["/usr/bin/pgcat"] From 8025ddd9f133ef510be23842b5472569ee398150 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 11 Feb 2022 22:19:49 -0800 Subject: [PATCH 076/551] Pass real server info to the client (#10) --- src/client.rs | 3 ++- src/config.rs | 5 +++++ src/main.rs | 12 +++++++++++- src/messages.rs | 12 +++++------- src/pool.rs | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/client.rs b/src/client.rs index a9e7a6076..154716c7b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -63,6 +63,7 @@ impl Client { client_server_map: ClientServerMap, transaction_mode: bool, default_server_role: Option, + server_info: BytesMut, ) -> Result { loop { // Could be StartupMessage or SSLRequest @@ -102,7 +103,7 @@ impl Client { let secret_key: i32 = rand::random(); auth_ok(&mut stream).await?; - server_parameters(&mut stream).await?; + write_all(&mut stream, server_info).await?; backend_key_data(&mut stream, process_id, secret_key).await?; ready_for_query(&mut stream).await?; diff --git a/src/config.rs b/src/config.rs index 79c2c0b3a..d2e050b0f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -90,6 +90,11 @@ pub async fn parse(path: &str) -> Result { let mut dup_check = HashSet::new(); let mut primary_count = 0; + if shard.1.servers.len() == 0 { + println!("> Shard {} has no servers configured", shard.0); + return Err(Error::BadConfig); + } + for server in &shard.1.servers { dup_check.insert(server); diff --git a/src/main.rs b/src/main.rs index 05903d661..37f820f07 100644 --- a/src/main.rs +++ b/src/main.rs @@ -86,7 +86,7 @@ async fn main() { ); println!("> Connection timeout: {}ms", config.general.connect_timeout); - let pool = ConnectionPool::from_config(config.clone(), client_server_map.clone()).await; + let mut pool = ConnectionPool::from_config(config.clone(), client_server_map.clone()).await; let transaction_mode = config.general.pool_mode == "transaction"; let default_server_role = match config.query_router.default_role.as_ref() { "any" => None, @@ -98,11 +98,20 @@ async fn main() { } }; + let server_info = match pool.validate().await { + Ok(info) => info, + Err(err) => { + println!("> Could not validate connection pool: {:?}", err); + return; + } + }; + println!("> Waiting for clients..."); loop { let pool = pool.clone(); let client_server_map = client_server_map.clone(); + let server_info = server_info.clone(); let (socket, addr) = match listener.accept().await { Ok((socket, addr)) => (socket, addr), @@ -124,6 +133,7 @@ async fn main() { client_server_map, transaction_mode, default_server_role, + server_info, ) .await { diff --git a/src/messages.rs b/src/messages.rs index 5f17d8d5d..cd99de75b 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -8,10 +8,8 @@ use crate::errors::Error; // This is a funny one. `psql` parses this to figure out which // queries to send when using shortcuts, e.g. \d+. -// -// TODO: Actually get the version from the server itself. -// -const SERVER_VESION: &str = "12.9 (Ubuntu 12.9-0ubuntu0.20.04.1)"; +// No longer used. Keeping it here until I'm sure we don't need it again. +const _SERVER_VESION: &str = "12.9 (Ubuntu 12.9-0ubuntu0.20.04.1)"; /// Tell the client that authentication handshake completed successfully. pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { @@ -27,12 +25,12 @@ pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { /// Send server parameters to the client. This will tell the client /// what server version and what's the encoding we're using. // -// TODO: Forward these from the server instead of hardcoding. +// No longer used. Keeping it here until I'm sure we don't need it again. // -pub async fn server_parameters(stream: &mut TcpStream) -> Result<(), Error> { +pub async fn _server_parameters(stream: &mut TcpStream) -> Result<(), Error> { let client_encoding = BytesMut::from(&b"client_encoding\0UTF8\0"[..]); let server_version = - BytesMut::from(&format!("server_version\0{}\0", SERVER_VESION).as_bytes()[..]); + BytesMut::from(&format!("server_version\0{}\0", _SERVER_VESION).as_bytes()[..]); // Client encoding let len = client_encoding.len() as i32 + 4; // TODO: add more parameters here diff --git a/src/pool.rs b/src/pool.rs index 8c92bc6d9..343233072 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,6 +1,7 @@ /// Pooling and failover and banlist. use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection}; +use bytes::BytesMut; use chrono::naive::NaiveDateTime; use crate::config::{Address, Config, Role, User}; @@ -105,6 +106,38 @@ impl ConnectionPool { } } + /// Connect to all shards and grab server information. + /// Return server information we will pass to the clients + /// when they connect. + pub async fn validate(&mut self) -> Result { + let mut server_infos = Vec::new(); + + for shard in 0..self.shards() { + // TODO: query all primary and replicas in the shard configuration. + let connection = match self.get(Some(shard), None).await { + Ok(conn) => conn, + Err(err) => { + println!("> Shard {} down or misconfigured.", shard); + return Err(err); + } + }; + + let mut proxy = connection.0; + let _address = connection.1; + let server = &mut *proxy; + + server_infos.push(server.server_info()); + } + + // TODO: compare server information to make sure + // all shards are running identical configurations. + if server_infos.len() == 0 { + return Err(Error::AllServersDown); + } + + Ok(server_infos[0].clone()) + } + /// Get a connection from the pool. pub async fn get( &mut self, From 4bded7015d8cfd8c6c78daf4c73f7e0d24d2cd54 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 11 Feb 2022 22:24:27 -0800 Subject: [PATCH 077/551] extended protocol tests --- .circleci/run_tests.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 4307e42df..c24a59b43 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -13,12 +13,16 @@ psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.s pgbench -i -h 127.0.0.1 -p 6432 # Run it -pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 +pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol simple +# Extended protocol +pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended + +# Sharding insert psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_insert.sql +# Sharding select psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql > /dev/null -# psql -f tests/sharding/query_routing_test_validate.sql - -psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null \ No newline at end of file +# Replica/primary selection & more sharding tests +psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null From e52924754414e05ab85dbc80494c47996038f610 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 12 Feb 2022 09:24:24 -0800 Subject: [PATCH 078/551] print session duration; connect to all servers when validating (#11) --- src/main.rs | 53 ++++++++++++++++++++++++++++++++++++++++++++++++----- src/pool.rs | 31 +++++++++++++++++++------------ 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/src/main.rs b/src/main.rs index 37f820f07..58e677879 100644 --- a/src/main.rs +++ b/src/main.rs @@ -123,10 +123,9 @@ async fn main() { // Client goes to another thread, bye. tokio::task::spawn(async move { - println!( - ">> Client {:?} connected, transaction pooling: {}", - addr, transaction_mode - ); + let start = chrono::offset::Utc::now().naive_utc(); + + println!(">> Client {:?} connected", addr); match client::Client::startup( socket, @@ -142,7 +141,13 @@ async fn main() { match client.handle(pool).await { Ok(()) => { - println!(">> Client {:?} disconnected.", addr); + let duration = chrono::offset::Utc::now().naive_utc() - start; + + println!( + ">> Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); } Err(err) => { @@ -159,3 +164,41 @@ async fn main() { }); } } + +/// Format chrono::Duration to be more human-friendly. +/// +/// # Arguments +/// +/// * `duration` - A duration of time +fn format_duration(duration: &chrono::Duration) -> String { + let seconds = { + let seconds = duration.num_seconds() % 60; + if seconds < 10 { + format!("0{}", seconds) + } else { + format!("{}", seconds) + } + }; + + let minutes = { + let minutes = duration.num_minutes() % 60; + if minutes < 10 { + format!("0{}", minutes) + } else { + format!("{}", minutes) + } + }; + + let hours = { + let hours = duration.num_hours() % 24; + if hours < 10 { + format!("0{}", hours) + } else { + format!("{}", hours) + } + }; + + let days = duration.num_days().to_string(); + + format!("{}d {}:{}:{}", days, hours, minutes, seconds) +} diff --git a/src/pool.rs b/src/pool.rs index 343233072..a4f4bb3bb 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -109,24 +109,27 @@ impl ConnectionPool { /// Connect to all shards and grab server information. /// Return server information we will pass to the clients /// when they connect. + /// This also warms up the pool for clients that connect when + /// the pooler starts up. pub async fn validate(&mut self) -> Result { let mut server_infos = Vec::new(); for shard in 0..self.shards() { - // TODO: query all primary and replicas in the shard configuration. - let connection = match self.get(Some(shard), None).await { - Ok(conn) => conn, - Err(err) => { - println!("> Shard {} down or misconfigured.", shard); - return Err(err); - } - }; + for _ in 0..self.replicas(shard) { + let connection = match self.get(Some(shard), None).await { + Ok(conn) => conn, + Err(err) => { + println!("> Shard {} down or misconfigured.", shard); + return Err(err); + } + }; - let mut proxy = connection.0; - let _address = connection.1; - let server = &mut *proxy; + let mut proxy = connection.0; + let _address = connection.1; + let server = &mut *proxy; - server_infos.push(server.server_info()); + server_infos.push(server.server_info()); + } } // TODO: compare server information to make sure @@ -326,6 +329,10 @@ impl ConnectionPool { pub fn shards(&self) -> usize { self.databases.len() } + + pub fn replicas(&self, shard: usize) -> usize { + self.addresses[shard].len() + } } pub struct ServerPool { From 51aae8c01799c93ebab4bdb930292ec686c464b6 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 12 Feb 2022 10:16:05 -0800 Subject: [PATCH 079/551] Cleaner shutdown (#12) * Cleaner shutdown * mark as bad just in case although im pretty sure we dont need it * server session duration * test clean shutdown * ah --- .circleci/config.yml | 4 +- .circleci/run_tests.sh | 3 ++ src/main.rs | 114 +++++++++++++++++++++++------------------ src/server.rs | 29 +++++++++++ 4 files changed, 97 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bd5b396bd..5feb5c583 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: key: cargo-lock-2-{{ checksum "Cargo.lock" }} - run: name: "Install dependencies" - command: "sudo apt-get update && sudo apt-get install -y postgresql-contrib-12 postgresql-client-12" + command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12" - run: name: "Build" command: "cargo build" @@ -47,4 +47,4 @@ jobs: workflows: build: jobs: - - build \ No newline at end of file + - build diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index c24a59b43..fc4482571 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -26,3 +26,6 @@ psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql > / # Replica/primary selection & more sharding tests psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null + +# Attempt clean shut down +killall pgcat -s SIGINT diff --git a/src/main.rs b/src/main.rs index 58e677879..29b0a2cb5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,7 @@ extern crate toml; use regex::Regex; use tokio::net::TcpListener; +use tokio::signal; use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -108,61 +109,72 @@ async fn main() { println!("> Waiting for clients..."); - loop { - let pool = pool.clone(); - let client_server_map = client_server_map.clone(); - let server_info = server_info.clone(); - - let (socket, addr) = match listener.accept().await { - Ok((socket, addr)) => (socket, addr), - Err(err) => { - println!("> Listener: {:?}", err); - continue; - } - }; - - // Client goes to another thread, bye. - tokio::task::spawn(async move { - let start = chrono::offset::Utc::now().naive_utc(); - - println!(">> Client {:?} connected", addr); - - match client::Client::startup( - socket, - client_server_map, - transaction_mode, - default_server_role, - server_info, - ) - .await - { - Ok(mut client) => { - println!(">> Client {:?} authenticated successfully!", addr); - - match client.handle(pool).await { - Ok(()) => { - let duration = chrono::offset::Utc::now().naive_utc() - start; - - println!( - ">> Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); - } - - Err(err) => { - println!(">> Client disconnected with error: {:?}", err); - client.release(); - } - } - } + // Main app runs here. + tokio::task::spawn(async move { + loop { + let pool = pool.clone(); + let client_server_map = client_server_map.clone(); + let server_info = server_info.clone(); + let (socket, addr) = match listener.accept().await { + Ok((socket, addr)) => (socket, addr), Err(err) => { - println!(">> Error: {:?}", err); + println!("> Listener: {:?}", err); + continue; } }; - }); - } + + // Client goes to another thread, bye. + tokio::task::spawn(async move { + let start = chrono::offset::Utc::now().naive_utc(); + + println!(">> Client {:?} connected", addr); + + match client::Client::startup( + socket, + client_server_map, + transaction_mode, + default_server_role, + server_info, + ) + .await + { + Ok(mut client) => { + println!(">> Client {:?} authenticated successfully!", addr); + + match client.handle(pool).await { + Ok(()) => { + let duration = chrono::offset::Utc::now().naive_utc() - start; + + println!( + ">> Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + } + + Err(err) => { + println!(">> Client disconnected with error: {:?}", err); + client.release(); + } + } + } + + Err(err) => { + println!(">> Error: {:?}", err); + } + }; + }); + } + }); + + // Setup shut down sequence + match signal::ctrl_c().await { + Ok(()) => {} + Err(err) => { + eprintln!("Unable to listen for shutdown signal: {}", err); + } + }; } /// Format chrono::Duration to be more human-friendly. diff --git a/src/server.rs b/src/server.rs index 13f362a5b..ef3c611a1 100644 --- a/src/server.rs +++ b/src/server.rs @@ -49,7 +49,11 @@ pub struct Server { // Mapping of clients and servers used for query cancellation. client_server_map: ClientServerMap, + // Server role, e.g. primary or replica. role: Role, + + // Server connected at + connected_at: chrono::naive::NaiveDateTime, } impl Server { @@ -193,6 +197,7 @@ impl Server { bad: false, client_server_map: client_server_map, role: role, + connected_at: chrono::offset::Utc::now().naive_utc(), }); } @@ -417,3 +422,27 @@ impl Server { } } } + +impl Drop for Server { + // Try to do a clean shut down. + fn drop(&mut self) { + let mut bytes = BytesMut::with_capacity(4); + bytes.put_u8(b'X'); + bytes.put_i32(4); + + match self.write.try_write(&bytes) { + Ok(n) => (), + Err(_) => (), + }; + + self.bad = true; + + let now = chrono::offset::Utc::now().naive_utc(); + let duration = now - self.connected_at; + + println!( + ">> Server connection closed, session duration: {}", + crate::format_duration(&duration) + ); + } +} From 51060c8b49ae2250f427b4641224df8e216104f2 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 14 Feb 2022 05:11:53 -0800 Subject: [PATCH 080/551] parse startup client parameters (#16) --- src/client.rs | 9 ++++++++- src/messages.rs | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/client.rs b/src/client.rs index 154716c7b..43b3e63b9 100644 --- a/src/client.rs +++ b/src/client.rs @@ -8,6 +8,8 @@ use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::net::TcpStream; +use std::collections::HashMap; + use crate::config::Role; use crate::errors::Error; use crate::messages::*; @@ -52,6 +54,9 @@ pub struct Client { // Unless client specifies, route queries to the servers that have this role, // e.g. primary or replicas or any. default_server_role: Option, + + // Client parameters, e.g. user, client_encoding, etc. + parameters: HashMap, } impl Client { @@ -96,7 +101,7 @@ impl Client { // Regular startup message. 196608 => { // TODO: perform actual auth. - // TODO: record startup parameters client sends over. + let parameters = parse_startup(bytes.clone())?; // Generate random backend ID and secret key let process_id: i32 = rand::random(); @@ -121,6 +126,7 @@ impl Client { secret_key: secret_key, client_server_map: client_server_map, default_server_role: default_server_role, + parameters: parameters, }); } @@ -141,6 +147,7 @@ impl Client { secret_key: secret_key, client_server_map: client_server_map, default_server_role: default_server_role, + parameters: HashMap::new(), }); } diff --git a/src/messages.rs b/src/messages.rs index cd99de75b..beb650520 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -1,9 +1,11 @@ -use bytes::{BufMut, BytesMut}; +use bytes::{Buf, BufMut, BytesMut}; use md5::{Digest, Md5}; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::net::TcpStream; +use std::collections::HashMap; + use crate::errors::Error; // This is a funny one. `psql` parses this to figure out which @@ -105,6 +107,51 @@ pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Resu } } +/// Parse StartupMessage parameters. +/// e.g. user, database, application_name, etc. +pub fn parse_startup(mut bytes: BytesMut) -> Result, Error> { + let mut result = HashMap::new(); + let mut buf = Vec::new(); + let mut tmp = String::new(); + + while bytes.has_remaining() { + let mut c = bytes.get_u8(); + + // Null-terminated C-strings. + while c != 0 { + tmp.push(c as char); + c = bytes.get_u8(); + } + + if tmp.len() > 0 { + buf.push(tmp.clone()); + tmp.clear(); + } + } + + // Expect pairs of name and value + // and at least one pair to be present. + if buf.len() % 2 != 0 && buf.len() >= 2 { + return Err(Error::ClientBadStartup); + } + + let mut i = 0; + while i < buf.len() { + let name = buf[i].clone(); + let value = buf[i + 1].clone(); + let _ = result.insert(name, value); + i += 2; + } + + // Minimum required parameters + // I want to have the user at the very minimum, according to the protocol spec. + if !result.contains_key("user") { + return Err(Error::ClientBadStartup); + } + + Ok(result) +} + /// Send password challenge response to the server. /// This is the MD5 challenge. pub async fn md5_password( From ccc474e589d80e78055a7492b33fb3bb8ccf391c Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 14 Feb 2022 06:36:05 -0800 Subject: [PATCH 081/551] TODOs (#18) --- CONTRIBUTING.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af38a647f..14714008f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,21 @@ +## Introduction + Thank you for contributing! Just a few tips here: 1. `cargo fmt` your code before opening up a PR 2. Run the "test suite" (i.e. PgBench) to make sure everything still works. Happy hacking! + +## TODOs + +A non-exhaustive list of things that would be useful to implement. + +#### Client authentication +MD5 is probably sufficient, but maybe others too. + +#### Statistics +Same as PgBouncer, e.g. client wait, transactions, timings, etc. I'm thinking we can use `mpsc` here ([docs](https://tokio.rs/tokio/tutorial/channels)), with clients sending stats and a task collecting and aggregating them. This should avoid atomics/mutexes. Caveat is the task should make sure not to crash, so the channels don't get backed up. + +#### Admin +Admin database for stats collection and pooler administration. PgBouncer gives us a nice example on how to do that, specifically how to implement `RowDescription` and `DataRow` messages, [example here](https://github.com/pgbouncer/pgbouncer/blob/4f9ced8e63d317a6ff45c8b0efa876b32161f6db/src/admin.c#L813). From 42c2af266e2da7be1dce19893132afc9df35a447 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 14 Feb 2022 10:00:55 -0800 Subject: [PATCH 082/551] Some stats (#19) --- Cargo.lock | 71 ++++++++++++++++++++-- Cargo.toml | 1 + src/client.rs | 53 +++++++++++----- src/main.rs | 29 ++++++++- src/pool.rs | 22 ++++++- src/server.rs | 11 +++- src/stats.rs | 163 ++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 325 insertions(+), 25 deletions(-) create mode 100644 src/stats.rs diff --git a/Cargo.lock b/Cargo.lock index 1f18149f8..5ddda15f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,6 +110,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures-channel" version = "0.3.19" @@ -325,11 +331,12 @@ dependencies = [ "md-5", "num_cpus", "once_cell", - "rand", + "rand 0.8.4", "regex", "serde", "serde_derive", "sha-1", + "statsd", "tokio", "toml", ] @@ -370,6 +377,29 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" +dependencies = [ + "libc", + "rand 0.4.6", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.8.4" @@ -378,7 +408,7 @@ checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.3", "rand_hc", ] @@ -389,9 +419,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", ] +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.3" @@ -407,7 +452,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" dependencies = [ - "rand_core", + "rand_core 0.6.3", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", ] [[package]] @@ -491,6 +545,15 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "statsd" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181f817969f3b9f76f70866611c0555b5537f8fdbf14cbdadebb9b54155bd5a" +dependencies = [ + "rand 0.3.23", +] + [[package]] name = "syn" version = "1.0.86" diff --git a/Cargo.toml b/Cargo.toml index 6db820ca3..8e89b4987 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,4 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" +statsd = "0.14" diff --git a/src/client.rs b/src/client.rs index 43b3e63b9..cbb98727a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -16,6 +16,7 @@ use crate::messages::*; use crate::pool::{ClientServerMap, ConnectionPool}; use crate::server::Server; use crate::sharding::Sharder; +use crate::stats::Reporter; pub const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+';"; pub const ROLE_REGEX: &str = r"SET SERVER ROLE TO '(PRIMARY|REPLICA)';"; @@ -57,6 +58,9 @@ pub struct Client { // Client parameters, e.g. user, client_encoding, etc. parameters: HashMap, + + // Statistics + stats: Reporter, } impl Client { @@ -69,6 +73,7 @@ impl Client { transaction_mode: bool, default_server_role: Option, server_info: BytesMut, + stats: Reporter, ) -> Result { loop { // Could be StartupMessage or SSLRequest @@ -127,6 +132,7 @@ impl Client { client_server_map: client_server_map, default_server_role: default_server_role, parameters: parameters, + stats: stats, }); } @@ -148,6 +154,7 @@ impl Client { client_server_map: client_server_map, default_server_role: default_server_role, parameters: HashMap::new(), + stats: stats, }); } @@ -220,7 +227,6 @@ impl Client { }; // Grab a server from the pool. - // None = any shard let connection = match pool.get(shard, role).await { Ok(conn) => conn, Err(err) => { @@ -287,11 +293,19 @@ impl Client { } } - // Release server - if !server.in_transaction() && self.transaction_mode { - shard = None; - role = self.default_server_role; - break; + // Send statistic + self.stats.query(); + + // Transaction over + if !server.in_transaction() { + self.stats.transaction(); + + // Release server + if self.transaction_mode { + shard = None; + role = self.default_server_role; + break; + } } } @@ -350,11 +364,17 @@ impl Client { } } + self.stats.query(); + // Release server - if !server.in_transaction() && self.transaction_mode { - shard = None; - role = self.default_server_role; - break; + if !server.in_transaction() { + self.stats.transaction(); + + if self.transaction_mode { + shard = None; + role = self.default_server_role; + break; + } } } @@ -378,11 +398,14 @@ impl Client { }; // Release the server - if !server.in_transaction() && self.transaction_mode { - println!("Releasing after copy done"); - shard = None; - role = self.default_server_role; - break; + if !server.in_transaction() { + self.stats.transaction(); + + if self.transaction_mode { + shard = None; + role = self.default_server_role; + break; + } } } diff --git a/src/main.rs b/src/main.rs index 29b0a2cb5..b4d51ad0b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,7 @@ extern crate num_cpus; extern crate once_cell; extern crate serde; extern crate serde_derive; +extern crate statsd; extern crate tokio; extern crate toml; @@ -30,6 +31,7 @@ use tokio::signal; use std::collections::HashMap; use std::sync::{Arc, Mutex}; +use tokio::sync::mpsc; mod client; mod config; @@ -38,11 +40,13 @@ mod messages; mod pool; mod server; mod sharding; +mod stats; // Support for query cancellation: this maps our process_ids and // secret keys to the backend's. use config::Role; use pool::{ClientServerMap, ConnectionPool}; +use stats::{Collector, Reporter}; /// Main! #[tokio::main(worker_threads = 4)] @@ -87,7 +91,23 @@ async fn main() { ); println!("> Connection timeout: {}ms", config.general.connect_timeout); - let mut pool = ConnectionPool::from_config(config.clone(), client_server_map.clone()).await; + // Collect statistics and send them to StatsD + let (tx, rx) = mpsc::channel(100); + + tokio::task::spawn(async move { + println!("> Statistics reporter started"); + + let mut stats_collector = Collector::new(rx); + stats_collector.collect().await; + }); + + let mut pool = ConnectionPool::from_config( + config.clone(), + client_server_map.clone(), + Reporter::new(tx.clone()), + ) + .await; + let transaction_mode = config.general.pool_mode == "transaction"; let default_server_role = match config.query_router.default_role.as_ref() { "any" => None, @@ -115,6 +135,7 @@ async fn main() { let pool = pool.clone(); let client_server_map = client_server_map.clone(); let server_info = server_info.clone(); + let reporter = Reporter::new(tx.clone()); let (socket, addr) = match listener.accept().await { Ok((socket, addr)) => (socket, addr), @@ -136,6 +157,7 @@ async fn main() { transaction_mode, default_server_role, server_info, + reporter, ) .await { @@ -170,7 +192,10 @@ async fn main() { // Setup shut down sequence match signal::ctrl_c().await { - Ok(()) => {} + Ok(()) => { + println!("> Shutting down..."); + } + Err(err) => { eprintln!("Unable to listen for shutdown signal: {}", err); } diff --git a/src/pool.rs b/src/pool.rs index a4f4bb3bb..62710eb38 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -7,6 +7,7 @@ use chrono::naive::NaiveDateTime; use crate::config::{Address, Config, Role, User}; use crate::errors::Error; use crate::server::Server; +use crate::stats::Reporter; use std::collections::HashMap; use std::sync::{ @@ -14,6 +15,7 @@ use std::sync::{ Arc, Mutex, }; +use std::time::Instant; // Banlist: bad servers go in here. pub type BanList = Arc>>>; @@ -29,11 +31,16 @@ pub struct ConnectionPool { healthcheck_timeout: u64, ban_time: i64, pool_size: u32, + stats: Reporter, } impl ConnectionPool { /// Construct the connection pool from a config file. - pub async fn from_config(config: Config, client_server_map: ClientServerMap) -> ConnectionPool { + pub async fn from_config( + config: Config, + client_server_map: ClientServerMap, + stats: Reporter, + ) -> ConnectionPool { let mut shards = Vec::new(); let mut addresses = Vec::new(); let mut banlist = Vec::new(); @@ -71,6 +78,7 @@ impl ConnectionPool { config.user.clone(), &shard.database, client_server_map.clone(), + stats.clone(), ); let pool = Pool::builder() @@ -103,6 +111,7 @@ impl ConnectionPool { healthcheck_timeout: config.general.healthcheck_timeout, ban_time: config.general.ban_time, pool_size: config.general.pool_size, + stats: stats, } } @@ -149,6 +158,7 @@ impl ConnectionPool { ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { // Set this to false to gain ~3-4% speed. let with_health_check = true; + let now = Instant::now(); let shard = match shard { Some(shard) => shard, @@ -227,6 +237,7 @@ impl ConnectionPool { }; if !with_health_check { + self.stats.checkout_time(now.elapsed().as_millis()); return Ok((conn, address.clone())); } @@ -241,7 +252,10 @@ impl ConnectionPool { { // Check if health check succeeded Ok(res) => match res { - Ok(_) => return Ok((conn, address.clone())), + Ok(_) => { + self.stats.checkout_time(now.elapsed().as_millis()); + return Ok((conn, address.clone())); + } Err(_) => { println!( ">> Banning replica {} because of failed health check", @@ -340,6 +354,7 @@ pub struct ServerPool { user: User, database: String, client_server_map: ClientServerMap, + stats: Reporter, } impl ServerPool { @@ -348,12 +363,14 @@ impl ServerPool { user: User, database: &str, client_server_map: ClientServerMap, + stats: Reporter, ) -> ServerPool { ServerPool { address: address, user: user, database: database.to_string(), client_server_map: client_server_map, + stats: stats, } } } @@ -375,6 +392,7 @@ impl ManageConnection for ServerPool { &self.database, self.client_server_map.clone(), self.address.role, + self.stats.clone(), ) .await } diff --git a/src/server.rs b/src/server.rs index ef3c611a1..ce55d9cae 100644 --- a/src/server.rs +++ b/src/server.rs @@ -11,6 +11,7 @@ use tokio::net::TcpStream; use crate::config::{Address, Role}; use crate::errors::Error; use crate::messages::*; +use crate::stats::Reporter; use crate::ClientServerMap; /// Server state. @@ -54,6 +55,9 @@ pub struct Server { // Server connected at connected_at: chrono::naive::NaiveDateTime, + + // Stats + stats: Reporter, } impl Server { @@ -67,6 +71,7 @@ impl Server { database: &str, client_server_map: ClientServerMap, role: Role, + stats: Reporter, ) -> Result { let mut stream = match TcpStream::connect(&format!("{}:{}", host, port)).await { Ok(stream) => stream, @@ -198,6 +203,7 @@ impl Server { client_server_map: client_server_map, role: role, connected_at: chrono::offset::Utc::now().naive_utc(), + stats: stats, }); } @@ -236,6 +242,8 @@ impl Server { /// Send data to the server from the client. pub async fn send(&mut self, messages: BytesMut) -> Result<(), Error> { + self.stats.data_sent(messages.len()); + match write_all_half(&mut self.write, messages).await { Ok(_) => Ok(()), Err(err) => { @@ -280,8 +288,6 @@ impl Server { self.in_transaction = false; } - // Error client didn't clean up! - // We shuold drop this server 'E' => { self.in_transaction = true; } @@ -332,6 +338,7 @@ impl Server { } let bytes = self.buffer.clone(); + self.stats.data_received(bytes.len()); self.buffer.clear(); Ok(bytes) diff --git a/src/stats.rs b/src/stats.rs new file mode 100644 index 000000000..a7eecec0a --- /dev/null +++ b/src/stats.rs @@ -0,0 +1,163 @@ +use statsd::Client; +/// Statistics collector and publisher. +use tokio::sync::mpsc::{Receiver, Sender}; + +use std::collections::HashMap; +use std::time::Instant; + +#[derive(Debug)] +pub enum StatisticName { + CheckoutTime, + //QueryRuntime, + //TransactionTime, + Queries, + Transactions, + DataSent, + DataReceived, +} + +#[derive(Debug)] +pub struct Statistic { + pub name: StatisticName, + pub value: i64, +} + +#[derive(Clone, Debug)] +pub struct Reporter { + tx: Sender, +} + +impl Reporter { + pub fn new(tx: Sender) -> Reporter { + Reporter { tx: tx } + } + + pub fn query(&mut self) { + let statistic = Statistic { + name: StatisticName::Queries, + value: 1, + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn transaction(&mut self) { + let statistic = Statistic { + name: StatisticName::Transactions, + value: 1, + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn data_sent(&mut self, amount: usize) { + let statistic = Statistic { + name: StatisticName::DataSent, + value: amount as i64, + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn data_received(&mut self, amount: usize) { + let statistic = Statistic { + name: StatisticName::DataReceived, + value: amount as i64, + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn checkout_time(&mut self, ms: u128) { + let statistic = Statistic { + name: StatisticName::CheckoutTime, + value: ms as i64, + }; + + let _ = self.tx.try_send(statistic); + } +} + +pub struct Collector { + rx: Receiver, + client: Client, +} + +impl Collector { + pub fn new(rx: Receiver) -> Collector { + Collector { + rx: rx, + client: Client::new("127.0.0.1:8125", "pgcat").unwrap(), + } + } + + pub async fn collect(&mut self) { + let mut stats = HashMap::from([ + ("queries", 0), + ("transactions", 0), + ("data_sent", 0), + ("data_received", 0), + ("checkout_time", 0), + ]); + let mut now = Instant::now(); + + loop { + let stat = match self.rx.recv().await { + Some(stat) => stat, + None => { + println!(">> Statistics collector is shutting down."); + return; + } + }; + + // Some are counters, some are gauges... + match stat.name { + StatisticName::Queries => { + let counter = stats.entry("queries").or_insert(0); + *counter += stat.value; + } + + StatisticName::Transactions => { + let counter = stats.entry("transactions").or_insert(0); + *counter += stat.value; + } + + StatisticName::DataSent => { + let counter = stats.entry("data_sent").or_insert(0); + *counter += stat.value; + } + + StatisticName::DataReceived => { + let counter = stats.entry("data_received").or_insert(0); + *counter += stat.value; + } + + StatisticName::CheckoutTime => { + let counter = stats.entry("checkout_time").or_insert(0); + + // Report max time here + if stat.value > *counter { + *counter = stat.value; + } + } + }; + + // It's been 15 seconds. If there is no traffic, it won't publish anything, + // but it also doesn't matter then. + if now.elapsed().as_secs() > 15 { + let mut pipeline = self.client.pipeline(); + + println!(">> Publishing statistics to StatsD: {:?}", stats); + + for (key, value) in stats.iter_mut() { + pipeline.gauge(key, *value as f64); + *value = 0; + } + + pipeline.send(&self.client); + + now = Instant::now(); + } + } + } +} From 8f3241eb3cddaadb5b9e8cffed0680339e099aad Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 14 Feb 2022 10:27:32 -0800 Subject: [PATCH 083/551] bump statsd --- Cargo.lock | 67 ++++++------------------------------------------------ Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ddda15f7..1673d3a77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,12 +110,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "futures-channel" version = "0.3.19" @@ -331,7 +325,7 @@ dependencies = [ "md-5", "num_cpus", "once_cell", - "rand 0.8.4", + "rand", "regex", "serde", "serde_derive", @@ -377,29 +371,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" -dependencies = [ - "libc", - "rand 0.4.6", -] - -[[package]] -name = "rand" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" -dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", -] - [[package]] name = "rand" version = "0.8.4" @@ -408,7 +379,7 @@ checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", "rand_chacha", - "rand_core 0.6.3", + "rand_core", "rand_hc", ] @@ -419,24 +390,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core", ] -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.6.3" @@ -452,16 +408,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" dependencies = [ - "rand_core 0.6.3", -] - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", + "rand_core", ] [[package]] @@ -547,11 +494,11 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "statsd" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181f817969f3b9f76f70866611c0555b5537f8fdbf14cbdadebb9b54155bd5a" +checksum = "df1efceb4bf2c0b5ebec94354285a43bbbed1375605bdf2ebe4132299434a330" dependencies = [ - "rand 0.3.23", + "rand", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8e89b4987..0db9ecdd0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,4 +20,4 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -statsd = "0.14" +statsd = "0.15" From 44ed1be07a1ed074c739eda65af0938996d034ff Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 15 Feb 2022 08:18:01 -0800 Subject: [PATCH 084/551] More statistics (#20) * cleaner stats * remove shard selection there --- src/client.rs | 7 ++++ src/config.rs | 1 + src/pool.rs | 19 +++++---- src/server.rs | 54 ++++++++++--------------- src/stats.rs | 106 +++++++++++++++++++++++++++++++++++++++++++++----- 5 files changed, 136 insertions(+), 51 deletions(-) diff --git a/src/client.rs b/src/client.rs index cbb98727a..dd057869c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -57,6 +57,7 @@ pub struct Client { default_server_role: Option, // Client parameters, e.g. user, client_encoding, etc. + #[allow(dead_code)] parameters: HashMap, // Statistics @@ -302,8 +303,11 @@ impl Client { // Release server if self.transaction_mode { + self.stats.client_idle(); + shard = None; role = self.default_server_role; + break; } } @@ -371,8 +375,11 @@ impl Client { self.stats.transaction(); if self.transaction_mode { + self.stats.client_idle(); + shard = None; role = self.default_server_role; + break; } } diff --git a/src/config.rs b/src/config.rs index d2e050b0f..1a3f22b50 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,6 +17,7 @@ pub enum Role { pub struct Address { pub host: String, pub port: String, + pub shard: usize, pub role: Role, } diff --git a/src/pool.rs b/src/pool.rs index 62710eb38..3cf0c4e04 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -57,7 +57,7 @@ impl ConnectionPool { let mut pools = Vec::new(); let mut replica_addresses = Vec::new(); - for server in &shard.servers { + for (idx, server) in shard.servers.iter().enumerate() { let role = match server.2.as_ref() { "primary" => Role::Primary, "replica" => Role::Replica, @@ -71,6 +71,7 @@ impl ConnectionPool { host: server.0.clone(), port: server.1.to_string(), role: role, + shard: idx, }; let manager = ServerPool::new( @@ -165,6 +166,9 @@ impl ConnectionPool { None => 0, // TODO: pick a shard at random }; + // We are waiting for a server now. + self.stats.client_waiting(); + let addresses = &self.addresses[shard]; // Make sure if a specific role is requested, it's available in the pool. @@ -237,7 +241,8 @@ impl ConnectionPool { }; if !with_health_check { - self.stats.checkout_time(now.elapsed().as_millis()); + self.stats.checkout_time(now.elapsed().as_micros()); + self.stats.client_active(); return Ok((conn, address.clone())); } @@ -253,7 +258,8 @@ impl ConnectionPool { // Check if health check succeeded Ok(res) => match res { Ok(_) => { - self.stats.checkout_time(now.elapsed().as_millis()); + self.stats.checkout_time(now.elapsed().as_micros()); + self.stats.client_active(); return Ok((conn, address.clone())); } Err(_) => { @@ -385,13 +391,10 @@ impl ManageConnection for ServerPool { println!(">> Creating a new connection for the pool"); Server::startup( - &self.address.host, - &self.address.port, - &self.user.name, - &self.user.password, + &self.address, + &self.user, &self.database, self.client_server_map.clone(), - self.address.role, self.stats.clone(), ) .await diff --git a/src/server.rs b/src/server.rs index ce55d9cae..c1cacfc36 100644 --- a/src/server.rs +++ b/src/server.rs @@ -8,7 +8,7 @@ use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::net::TcpStream; -use crate::config::{Address, Role}; +use crate::config::{Address, User}; use crate::errors::Error; use crate::messages::*; use crate::stats::Reporter; @@ -16,11 +16,9 @@ use crate::ClientServerMap; /// Server state. pub struct Server { - // Server host, e.g. localhost - host: String, - - // Server port: e.g. 5432 - port: String, + // Server host, e.g. localhost, + // port, e.g. 5432, and role, e.g. primary or replica. + address: Address, // Buffered read socket read: BufReader, @@ -50,9 +48,6 @@ pub struct Server { // Mapping of clients and servers used for query cancellation. client_server_map: ClientServerMap, - // Server role, e.g. primary or replica. - role: Role, - // Server connected at connected_at: chrono::naive::NaiveDateTime, @@ -64,25 +59,23 @@ impl Server { /// Pretend to be the Postgres client and connect to the server given host, port and credentials. /// Perform the authentication and return the server in a ready-for-query mode. pub async fn startup( - host: &str, - port: &str, - user: &str, - password: &str, + address: &Address, + user: &User, database: &str, client_server_map: ClientServerMap, - role: Role, stats: Reporter, ) -> Result { - let mut stream = match TcpStream::connect(&format!("{}:{}", host, port)).await { - Ok(stream) => stream, - Err(err) => { - println!(">> Could not connect to server: {}", err); - return Err(Error::SocketError); - } - }; + let mut stream = + match TcpStream::connect(&format!("{}:{}", &address.host, &address.port)).await { + Ok(stream) => stream, + Err(err) => { + println!(">> Could not connect to server: {}", err); + return Err(Error::SocketError); + } + }; // Send the startup packet. - startup(&mut stream, user, database).await?; + startup(&mut stream, &user.name, database).await?; let mut server_info = BytesMut::with_capacity(25); let mut backend_id: i32 = 0; @@ -117,7 +110,8 @@ impl Server { Err(_) => return Err(Error::SocketError), }; - md5_password(&mut stream, user, password, &salt[..]).await?; + md5_password(&mut stream, &user.name, &user.password, &salt[..]) + .await?; } // Authentication handshake complete. @@ -189,8 +183,7 @@ impl Server { let (read, write) = stream.into_split(); return Ok(Server { - host: host.to_string(), - port: port.to_string(), + address: address.clone(), read: BufReader::new(read), write: write, buffer: BytesMut::with_capacity(8196), @@ -201,7 +194,6 @@ impl Server { data_available: false, bad: false, client_server_map: client_server_map, - role: role, connected_at: chrono::offset::Utc::now().naive_utc(), stats: stats, }); @@ -382,8 +374,8 @@ impl Server { ( self.backend_id, self.secret_key, - self.host.clone(), - self.port.clone(), + self.address.host.clone(), + self.address.port.clone(), ), ); } @@ -422,11 +414,7 @@ impl Server { } pub fn address(&self) -> Address { - Address { - host: self.host.to_string(), - port: self.port.to_string(), - role: self.role, - } + self.address.clone() } } diff --git a/src/stats.rs b/src/stats.rs index a7eecec0a..3f93b3f2b 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -14,6 +14,9 @@ pub enum StatisticName { Transactions, DataSent, DataReceived, + ClientsWaiting, + ClientsActive, + ClientsIdle, } #[derive(Debug)] @@ -76,6 +79,54 @@ impl Reporter { let _ = self.tx.try_send(statistic); } + + pub fn client_waiting(&mut self) { + let statistic = Statistic { + name: StatisticName::ClientsWaiting, + value: 1, + }; + + let _ = self.tx.try_send(statistic); + + let statistic = Statistic { + name: StatisticName::ClientsIdle, + value: -1, + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn client_active(&mut self) { + let statistic = Statistic { + name: StatisticName::ClientsWaiting, + value: -1, + }; + + let _ = self.tx.try_send(statistic); + + let statistic = Statistic { + name: StatisticName::ClientsActive, + value: 1, + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn client_idle(&mut self) { + let statistic = Statistic { + name: StatisticName::ClientsActive, + value: -1, + }; + + let _ = self.tx.try_send(statistic); + + let statistic = Statistic { + name: StatisticName::ClientsIdle, + value: 1, + }; + + let _ = self.tx.try_send(statistic); + } } pub struct Collector { @@ -93,12 +144,18 @@ impl Collector { pub async fn collect(&mut self) { let mut stats = HashMap::from([ - ("queries", 0), - ("transactions", 0), - ("data_sent", 0), - ("data_received", 0), - ("checkout_time", 0), + ("total_query_count", 0), + ("total_xact_count", 0), + ("total_sent", 0), + ("total_received", 0), + ("total_wait_time", 0), + ("maxwait_us", 0), + ("maxwait", 0), + ("cl_waiting", 0), + ("cl_active", 0), + ("cl_idle", 0), ]); + let mut now = Instant::now(); loop { @@ -113,32 +170,61 @@ impl Collector { // Some are counters, some are gauges... match stat.name { StatisticName::Queries => { - let counter = stats.entry("queries").or_insert(0); + let counter = stats.entry("total_query_count").or_insert(0); *counter += stat.value; } StatisticName::Transactions => { - let counter = stats.entry("transactions").or_insert(0); + let counter = stats.entry("total_xact_count").or_insert(0); *counter += stat.value; } StatisticName::DataSent => { - let counter = stats.entry("data_sent").or_insert(0); + let counter = stats.entry("total_sent").or_insert(0); *counter += stat.value; } StatisticName::DataReceived => { - let counter = stats.entry("data_received").or_insert(0); + let counter = stats.entry("total_received").or_insert(0); *counter += stat.value; } StatisticName::CheckoutTime => { - let counter = stats.entry("checkout_time").or_insert(0); + let counter = stats.entry("total_wait_time").or_insert(0); + *counter += stat.value; + + let counter = stats.entry("maxwait_us").or_insert(0); // Report max time here if stat.value > *counter { *counter = stat.value; } + + let counter = stats.entry("maxwait").or_insert(0); + let seconds = *counter / 1_000_000; + + if seconds > *counter { + *counter = seconds; + } + } + + StatisticName::ClientsActive => { + let counter = stats.entry("cl_active").or_insert(0); + + *counter += stat.value; + *counter = std::cmp::max(*counter, 0); + } + + StatisticName::ClientsWaiting => { + let counter = stats.entry("cl_waiting").or_insert(0); + *counter += stat.value; + *counter = std::cmp::max(*counter, 0); + } + + StatisticName::ClientsIdle => { + let counter = stats.entry("cl_idle").or_insert(0); + *counter += stat.value; + *counter = std::cmp::max(*counter, 0); } }; From c5baa0c02dca624c393128e4100f65352a450a8b Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 15 Feb 2022 13:10:32 -0800 Subject: [PATCH 085/551] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 050b1992a..66825e8aa 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,10 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su ## Local development 1. Install Rust (latest stable will work great). -2. `cargo run --release` (to get better benchmarks). +2. `cargo build --release` (to get better benchmarks). 3. Change the config in `pgcat.toml` to fit your setup (optional given next step). 4. Install Postgres and run `psql -f tests/sharding/query_routing_setup.sql` +5. `cargo run --release` You're ready to go! ### Tests From 2908e14d19946412ad7cda0eb176439dc048c48d Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 15 Feb 2022 13:11:24 -0800 Subject: [PATCH 086/551] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 66825e8aa..3f1c798ed 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su 1. Install Rust (latest stable will work great). 2. `cargo build --release` (to get better benchmarks). 3. Change the config in `pgcat.toml` to fit your setup (optional given next step). -4. Install Postgres and run `psql -f tests/sharding/query_routing_setup.sql` +4. Install Postgres and run `psql -f tests/sharding/query_routing_setup.sql` (user/password may be required depending on your setup) 5. `cargo run --release` You're ready to go! ### Tests From d2b3f102b69ce7d988844a53117e86cd461e7d1b Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 15 Feb 2022 22:45:45 -0800 Subject: [PATCH 087/551] Constants, comments, CI fixes, dead code clean-up (#21) * constants * server.rs docs * client.rs comments * dead code; comments * comment * query cancellation comments * remove unnecessary cast * move db setup up one step * query cancellation test * new line; good night --- .circleci/run_tests.sh | 14 +++- src/client.rs | 92 ++++++++++++++++++++------ src/constants.rs | 22 +++++++ src/main.rs | 1 + src/messages.rs | 40 ++---------- src/pool.rs | 7 +- src/server.rs | 145 ++++++++++++++++++++++++++--------------- src/sharding.rs | 2 +- 8 files changed, 208 insertions(+), 115 deletions(-) create mode 100644 src/constants.rs diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index fc4482571..2ff300827 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -3,12 +3,12 @@ set -e set -o xtrace +psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql + ./target/debug/pgcat & sleep 1 -psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql - # Setup PgBench pgbench -i -h 127.0.0.1 -p 6432 @@ -18,6 +18,13 @@ pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol simple # Extended protocol pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended +# COPY TO STDOUT test +psql -h 127.0.0.1 -p 6432 -c 'COPY (SELECT * FROM pgbench_accounts LIMIT 15) TO STDOUT;' > /dev/null + +# Query cancellation test +(psql -h 127.0.0.1 -p 6432 -c 'SELECT pg_sleep(5)' || true) & +killall psql -s SIGINT + # Sharding insert psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_insert.sql @@ -29,3 +36,6 @@ psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replic # Attempt clean shut down killall pgcat -s SIGINT + +# Allow for graceful shutdown +sleep 1 diff --git a/src/client.rs b/src/client.rs index dd057869c..0f3fd3179 100644 --- a/src/client.rs +++ b/src/client.rs @@ -5,12 +5,15 @@ use bytes::{Buf, BufMut, BytesMut}; use once_cell::sync::OnceCell; use regex::Regex; use tokio::io::{AsyncReadExt, BufReader}; -use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; -use tokio::net::TcpStream; +use tokio::net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, +}; use std::collections::HashMap; use crate::config::Role; +use crate::constants::*; use crate::errors::Error; use crate::messages::*; use crate::pool::{ClientServerMap, ConnectionPool}; @@ -97,7 +100,7 @@ impl Client { match code { // Client wants SSL. We don't support it at the moment. - 80877103 => { + SSL_REQUEST_CODE => { let mut no = BytesMut::with_capacity(1); no.put_u8(b'N'); @@ -105,7 +108,7 @@ impl Client { } // Regular startup message. - 196608 => { + PROTOCOL_VERSION_NUMBER => { // TODO: perform actual auth. let parameters = parse_startup(bytes.clone())?; @@ -138,7 +141,7 @@ impl Client { } // Query cancel request. - 80877102 => { + CANCEL_REQUEST_CODE => { let (read, write) = stream.into_split(); let process_id = bytes.get_i32(); @@ -168,23 +171,31 @@ impl Client { /// Client loop. We handle all messages between the client and the database here. pub async fn handle(&mut self, mut pool: ConnectionPool) -> Result<(), Error> { - // Special: cancelling existing running query + // The client wants to cancel a query it has issued previously. if self.cancel_mode { let (process_id, secret_key, address, port) = { let guard = self.client_server_map.lock().unwrap(); + match guard.get(&(self.process_id, self.secret_key)) { // Drop the mutex as soon as possible. + // We found the server the client is using for its query + // that it wants to cancel. Some((process_id, secret_key, address, port)) => ( process_id.clone(), secret_key.clone(), address.clone(), port.clone(), ), + + // The client doesn't know / got the wrong server, + // we're closing the connection for security reasons. None => return Ok(()), } }; - // TODO: pass actual server host and port somewhere. + // Opens a new separate connection to the server, sends the backend_id + // and secret_key and then closes it for security reasons. No other interactions + // take place. return Ok(Server::cancel(&address, &port, process_id, secret_key).await?); } @@ -217,7 +228,7 @@ impl Client { }; // Parse for special server role selection command. - // + // SET SERVER ROLE TO '(primary|replica)'; match self.select_role(message.clone()) { Some(r) => { custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; @@ -236,15 +247,17 @@ impl Client { } }; - let mut proxy = connection.0; + let mut reference = connection.0; let _address = connection.1; - let server = &mut *proxy; + let server = &mut *reference; // Claim this server as mine for query cancellation. server.claim(self.process_id, self.secret_key); + // Transaction loop. Multiple queries can be issued by the client here. + // The connection belongs to the client until the transaction is over, + // or until the client disconnects if we are in session mode. loop { - // No messages in the buffer, read one. let mut message = if message.len() == 0 { match read_message(&mut self.read).await { Ok(message) => message, @@ -268,19 +281,26 @@ impl Client { msg }; - let original = message.clone(); // To be forwarded to the server + // The message will be forwarded to the server intact. We still would like to + // parse it below to figure out what to do with it. + let original = message.clone(); + let code = message.get_u8() as char; let _len = message.get_i32() as usize; match code { + // ReadyForQuery 'Q' => { // TODO: implement retries here for read-only transactions. server.send(original).await?; + // Read all data the server has to offer, which can be multiple messages + // buffered in 8196 bytes chunks. loop { // TODO: implement retries here for read-only transactions. let response = server.recv().await?; + // Send server reply to the client. match write_all_half(&mut self.write, response).await { Ok(_) => (), Err(err) => { @@ -294,15 +314,18 @@ impl Client { } } - // Send statistic + // Report query executed statistics. self.stats.query(); - // Transaction over + // The transaction is over, we can release the connection back to the pool. if !server.in_transaction() { + // Report transaction executed statistics. self.stats.transaction(); - // Release server + // Release server back to the pool if we are in transaction mode. + // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { + // Report this client as idle. self.stats.client_idle(); shard = None; @@ -313,6 +336,7 @@ impl Client { } } + // Terminate 'X' => { // Client closing. Rollback and clean up // connection before releasing into the pool. @@ -326,35 +350,46 @@ impl Client { return Ok(()); } + // Parse + // The query with placeholders is here, e.g. `SELECT * FROM users WHERE email = $1 AND active = $2`. 'P' => { - // Extended protocol, let's buffer most of it self.buffer.put(&original[..]); } + // Bind + // The placeholder's replacements are here, e.g. 'user@email.com' and 'true' 'B' => { self.buffer.put(&original[..]); } // Describe + // Command a client can issue to describe a previously prepared named statement. 'D' => { self.buffer.put(&original[..]); } + // Execute + // Execute a prepared statement prepared in `P` and bound in `B`. 'E' => { self.buffer.put(&original[..]); } + // Sync + // Frontend (client) is asking for the query result now. 'S' => { - // Extended protocol, client requests sync self.buffer.put(&original[..]); - // TODO: retries for read-only transactions + // TODO: retries for read-only transactions. server.send(self.buffer.clone()).await?; + self.buffer.clear(); + // Read all data the server has to offer, which can be multiple messages + // buffered in 8196 bytes chunks. loop { // TODO: retries for read-only transactions let response = server.recv().await?; + match write_all_half(&mut self.write, response).await { Ok(_) => (), Err(err) => { @@ -368,9 +403,11 @@ impl Client { } } + // Report query executed statistics. self.stats.query(); - // Release server + // Release server back to the pool if we are in transaction mode. + // If we are in session mode, we keep the server until the client disconnects. if !server.in_transaction() { self.stats.transaction(); @@ -392,10 +429,13 @@ impl Client { server.send(original).await?; } + // CopyDone or CopyFail + // Copy is done, successfully or not. 'c' | 'f' => { - // Copy is done. server.send(original).await?; + let response = server.recv().await?; + match write_all_half(&mut self.write, response).await { Ok(_) => (), Err(err) => { @@ -404,24 +444,29 @@ impl Client { } }; - // Release the server + // Release server back to the pool if we are in transaction mode. + // If we are in session mode, we keep the server until the client disconnects. if !server.in_transaction() { self.stats.transaction(); if self.transaction_mode { shard = None; role = self.default_server_role; + break; } } } + // Some unexpected message. We either did not implement the protocol correctly + // or this is not a Postgres client we're talking to. _ => { println!(">>> Unexpected code: {}", code); } } } + // The server is no longer bound to us, we can't cancel it's queries anymore. self.release(); } } @@ -450,6 +495,7 @@ impl Client { let len = buf.get_i32(); let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); // Don't read the ternminating null + let rgx = match SHARDING_REGEX_RE.get() { Some(r) => r, None => return None, @@ -457,11 +503,13 @@ impl Client { if rgx.is_match(&query) { let shard = query.split("'").collect::>()[1]; + match shard.parse::() { Ok(shard) => { let sharder = Sharder::new(shards); Some(sharder.pg_bigint_hash(shard)) } + Err(_) => None, } } else { @@ -481,6 +529,7 @@ impl Client { let len = buf.get_i32(); let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); + let rgx = match ROLE_REGEX_RE.get() { Some(r) => r, None => return None, @@ -490,6 +539,7 @@ impl Client { // it'll be time to abstract :). if rgx.is_match(&query) { let role = query.split("'").collect::>()[1]; + match role { "PRIMARY" => Some(Role::Primary), "REPLICA" => Some(Role::Replica), diff --git a/src/constants.rs b/src/constants.rs new file mode 100644 index 000000000..3bdb47d8e --- /dev/null +++ b/src/constants.rs @@ -0,0 +1,22 @@ +/// Various protocol constants, as defined in +/// https://www.postgresql.org/docs/12/protocol-message-formats.html +/// and elsewhere in the source code. +/// Also other constants we use elsewhere. + +// Used in the StartupMessage to indicate regular handshake. +pub const PROTOCOL_VERSION_NUMBER: i32 = 196608; + +// SSLRequest: used to indicate we want an SSL connection. +pub const SSL_REQUEST_CODE: i32 = 80877103; + +// CancelRequest: the cancel request code. +pub const CANCEL_REQUEST_CODE: i32 = 80877102; + +// AuthenticationMD5Password +pub const MD5_ENCRYPTED_PASSWORD: i32 = 5; + +// AuthenticationOk +pub const AUTHENTICATION_SUCCESSFUL: i32 = 0; + +// ErrorResponse: A code identifying the field type; if zero, this is the message terminator and no string follows. +pub const MESSAGE_TERMINATOR: u8 = 0; diff --git a/src/main.rs b/src/main.rs index b4d51ad0b..324f9015e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,6 +35,7 @@ use tokio::sync::mpsc; mod client; mod config; +mod constants; mod errors; mod messages; mod pool; diff --git a/src/messages.rs b/src/messages.rs index beb650520..ec5ac66f7 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -1,18 +1,17 @@ +/// Helper functions to send one-off protocol messages +/// and handle TcpStream (TCP socket). use bytes::{Buf, BufMut, BytesMut}; use md5::{Digest, Md5}; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; -use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; -use tokio::net::TcpStream; +use tokio::net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, +}; use std::collections::HashMap; use crate::errors::Error; -// This is a funny one. `psql` parses this to figure out which -// queries to send when using shortcuts, e.g. \d+. -// No longer used. Keeping it here until I'm sure we don't need it again. -const _SERVER_VESION: &str = "12.9 (Ubuntu 12.9-0ubuntu0.20.04.1)"; - /// Tell the client that authentication handshake completed successfully. pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { let mut auth_ok = BytesMut::with_capacity(9); @@ -24,32 +23,6 @@ pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { Ok(write_all(stream, auth_ok).await?) } -/// Send server parameters to the client. This will tell the client -/// what server version and what's the encoding we're using. -// -// No longer used. Keeping it here until I'm sure we don't need it again. -// -pub async fn _server_parameters(stream: &mut TcpStream) -> Result<(), Error> { - let client_encoding = BytesMut::from(&b"client_encoding\0UTF8\0"[..]); - let server_version = - BytesMut::from(&format!("server_version\0{}\0", _SERVER_VESION).as_bytes()[..]); - - // Client encoding - let len = client_encoding.len() as i32 + 4; // TODO: add more parameters here - let mut res = BytesMut::with_capacity(64); - - res.put_u8(b'S'); - res.put_i32(len); - res.put_slice(&client_encoding[..]); - - let len = server_version.len() as i32 + 4; - res.put_u8(b'S'); - res.put_i32(len); - res.put_slice(&server_version[..]); - - Ok(write_all(stream, res).await?) -} - /// Give the client the process_id and secret we generated /// used in query cancellation. pub async fn backend_key_data( @@ -179,6 +152,7 @@ pub async fn md5_password( password.push(0); let mut message = BytesMut::with_capacity(password.len() as usize + 5); + message.put_u8(b'p'); message.put_i32(password.len() as i32 + 4); message.put_slice(&password[..]); diff --git a/src/pool.rs b/src/pool.rs index 3cf0c4e04..d348eec49 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -10,16 +10,11 @@ use crate::server::Server; use crate::stats::Reporter; use std::collections::HashMap; -use std::sync::{ - // atomic::{AtomicUsize, Ordering}, - Arc, - Mutex, -}; +use std::sync::{Arc, Mutex}; use std::time::Instant; // Banlist: bad servers go in here. pub type BanList = Arc>>>; -// pub type Counter = Arc; pub type ClientServerMap = Arc>>; #[derive(Clone, Debug)] diff --git a/src/server.rs b/src/server.rs index c1cacfc36..842b81fc7 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,14 +1,14 @@ -#![allow(dead_code)] -#![allow(unused_variables)] - ///! Implementation of the PostgreSQL server (database) protocol. ///! Here we are pretending to the a Postgres client. use bytes::{Buf, BufMut, BytesMut}; use tokio::io::{AsyncReadExt, BufReader}; -use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; -use tokio::net::TcpStream; +use tokio::net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, +}; use crate::config::{Address, User}; +use crate::constants::*; use crate::errors::Error; use crate::messages::*; use crate::stats::Reporter; @@ -20,23 +20,23 @@ pub struct Server { // port, e.g. 5432, and role, e.g. primary or replica. address: Address, - // Buffered read socket + // Buffered read socket. read: BufReader, - // Unbuffered write socket (our client code buffers) + // Unbuffered write socket (our client code buffers). write: OwnedWriteHalf, - // Our server response buffer + // Our server response buffer. We buffer data before we give it to the client. buffer: BytesMut, - // Server information the server sent us over on startup + // Server information the server sent us over on startup. server_info: BytesMut, // Backend id and secret key used for query cancellation. backend_id: i32, secret_key: i32, - // Is the server inside a transaction at the moment. + // Is the server inside a transaction or idle. in_transaction: bool, // Is there more data for the client to read. @@ -48,16 +48,16 @@ pub struct Server { // Mapping of clients and servers used for query cancellation. client_server_map: ClientServerMap, - // Server connected at + // Server connected at. connected_at: chrono::naive::NaiveDateTime, - // Stats + // Reports various metrics, e.g. data sent & received. stats: Reporter, } impl Server { /// Pretend to be the Postgres client and connect to the server given host, port and credentials. - /// Perform the authentication and return the server in a ready-for-query mode. + /// Perform the authentication and return the server in a ready for query state. pub async fn startup( address: &Address, user: &User, @@ -74,13 +74,15 @@ impl Server { } }; - // Send the startup packet. + // Send the startup packet telling the server we're a normal Postgres client. startup(&mut stream, &user.name, database).await?; - let mut server_info = BytesMut::with_capacity(25); + let mut server_info = BytesMut::new(); let mut backend_id: i32 = 0; let mut secret_key: i32 = 0; + // We'll be handling multiple packets, but they will all be structured the same. + // We'll loop here until this exchange is complete. loop { let code = match stream.read_u8().await { Ok(code) => code as char, @@ -93,16 +95,18 @@ impl Server { }; match code { + // Authentication 'R' => { - // Auth can proceed - let code = match stream.read_i32().await { - Ok(code) => code, + // Determine which kind of authentication is required, if any. + let auth_code = match stream.read_i32().await { + Ok(auth_code) => auth_code, Err(_) => return Err(Error::SocketError), }; - match code { - // MD5 - 5 => { + match auth_code { + MD5_ENCRYPTED_PASSWORD => { + // The salt is 4 bytes. + // See: https://www.postgresql.org/docs/12/protocol-message-formats.html let mut salt = vec![0u8; 4]; match stream.read_exact(&mut salt).await { @@ -114,16 +118,16 @@ impl Server { .await?; } - // Authentication handshake complete. - 0 => (), + AUTHENTICATION_SUCCESSFUL => (), _ => { - println!(">> Unsupported authentication mechanism: {}", code); + println!(">> Unsupported authentication mechanism: {}", auth_code); return Err(Error::ServerError); } } } + // ErrorResponse 'E' => { let error_code = match stream.read_u8().await { Ok(error_code) => error_code, @@ -131,46 +135,62 @@ impl Server { }; match error_code { - 0 => (), // Terminator + // No error message is present in the message. + MESSAGE_TERMINATOR => (), + + // An error message will be present. _ => { + // Read the error message without the terminating null character. let mut error = vec![0u8; len as usize - 4 - 1]; + match stream.read_exact(&mut error).await { Ok(_) => (), Err(_) => return Err(Error::SocketError), }; + // TODO: the error message contains multiple fields; we can decode them and + // present a prettier message to the user. + // See: https://www.postgresql.org/docs/12/protocol-error-fields.html println!(">> Server error: {}", String::from_utf8_lossy(&error)); } }; + return Err(Error::ServerError); } + // ParameterStatus 'S' => { - // Parameter let mut param = vec![0u8; len as usize - 4]; + match stream.read_exact(&mut param).await { Ok(_) => (), Err(_) => return Err(Error::SocketError), }; + // Save the parameter so we can pass it to the client later. + // These can be server_encoding, client_encoding, server timezone, Postgres version, + // and many more interesting things we should know about the Postgres server we are talking to. server_info.put_u8(b'S'); server_info.put_i32(len); server_info.put_slice(¶m[..]); } + // BackendKeyData 'K' => { - // Query cancellation data. + // The frontend must save these values if it wishes to be able to issue CancelRequest messages later. + // See: https://www.postgresql.org/docs/12/protocol-message-formats.html backend_id = match stream.read_i32().await { Ok(id) => id, - Err(err) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError), }; secret_key = match stream.read_i32().await { Ok(id) => id, - Err(err) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError), }; } + // ReadyForQuery 'Z' => { let mut idle = vec![0u8; len as usize - 4]; @@ -179,7 +199,8 @@ impl Server { Err(_) => return Err(Error::SocketError), }; - // Startup finished + // This is the last step in the client-server connection setup, + // and indicates the server is ready for to query it. let (read, write) = stream.into_split(); return Ok(Server { @@ -199,6 +220,8 @@ impl Server { }); } + // We have an unexpected message from the server during this exchange. + // Means we implemented the protocol wrong or we're not talking to a Postgres server. _ => { println!(">> Unknown code: {}", code); return Err(Error::ProtocolSyncError); @@ -207,7 +230,7 @@ impl Server { } } - /// Issue a cancellation request to the server. + /// Issue a query cancellation request to the server. /// Uses a separate connection that's not part of the connection pool. pub async fn cancel( host: &str, @@ -225,14 +248,14 @@ impl Server { let mut bytes = BytesMut::with_capacity(16); bytes.put_i32(16); - bytes.put_i32(80877102); + bytes.put_i32(CANCEL_REQUEST_CODE); bytes.put_i32(process_id); bytes.put_i32(secret_key); Ok(write_all(&mut stream, bytes).await?) } - /// Send data to the server from the client. + /// Send messages to the server from the client. pub async fn send(&mut self, messages: BytesMut) -> Result<(), Error> { self.stats.data_sent(messages.len()); @@ -246,7 +269,7 @@ impl Server { } } - /// Receive data from the server in response to a client request sent previously. + /// Receive data from the server in response to a client request. /// This method must be called multiple times while `self.is_data_available()` is true /// in order to receive all data the server has to offer. pub async fn recv(&mut self) -> Result { @@ -260,79 +283,90 @@ impl Server { } }; - // Buffer the message we'll forward to the client in a bit. + // Buffer the message we'll forward to the client later. self.buffer.put(&message[..]); let code = message.get_u8() as char; let _len = message.get_i32(); match code { + // ReadyForQuery 'Z' => { - // Ready for query, time to forward buffer to client. let transaction_state = message.get_u8() as char; match transaction_state { + // In transaction. 'T' => { self.in_transaction = true; } + // Idle, transaction over. 'I' => { self.in_transaction = false; } + // Some error occured, the transaction was rolled back. 'E' => { self.in_transaction = true; } + // Something totally unexpected, this is not a Postgres server we know. _ => { self.bad = true; return Err(Error::ProtocolSyncError); } }; + // There is no more data available from the server. self.data_available = false; break; } + // DataRow 'D' => { + // More data is available after this message, this is not the end of the reply. self.data_available = true; - // Don't flush yet, the more we buffer, the faster this goes. - // Up to a limit of course. + // Don't flush yet, the more we buffer, the faster this goes... + // up to a limit of course. if self.buffer.len() >= 8196 { break; } } - // CopyInResponse: copy is starting from client to server + // CopyInResponse: copy is starting from client to server. 'G' => break, - // CopyOutResponse: copy is starting from the server to the client + // CopyOutResponse: copy is starting from the server to the client. 'H' => { self.data_available = true; break; } - // CopyData + // CopyData: we are not buffering this one because there will be many more + // and we don't know how big this packet could be, best not to take a risk. 'd' => break, // CopyDone - 'c' => { - self.data_available = false; - // Buffer until ReadyForQuery shows up - } + // Buffer until ReadyForQuery shows up, so don't exit the loop yet. + 'c' => (), - _ => { - // Keep buffering, - } + // Anything else, e.g. errors, notices, etc. + // Keep buffering until ReadyForQuery shows up. + _ => (), }; } let bytes = self.buffer.clone(); + + // Keep track of how much data we got from the server for stats. self.stats.data_received(bytes.len()); + + // Clear the buffer for next query. self.buffer.clear(); + // Pass the data back to the client. Ok(bytes) } @@ -381,11 +415,11 @@ impl Server { } /// Execute an arbitrary query against the server. - /// It will use the Simple query protocol. + /// It will use the simple query protocol. /// Result will not be returned, so this is useful for things like `SET` or `ROLLBACK`. pub async fn query(&mut self, query: &str) -> Result<(), Error> { let mut query = BytesMut::from(&query.as_bytes()[..]); - query.put_u8(0); + query.put_u8(0); // C-string terminator (NULL character). let len = query.len() as i32 + 4; @@ -396,8 +430,10 @@ impl Server { msg.put_slice(&query[..]); self.send(msg).await?; + loop { let _ = self.recv().await?; + if !self.data_available { break; } @@ -407,26 +443,31 @@ impl Server { } /// A shorthand for `SET application_name = $1`. + #[allow(dead_code)] pub async fn set_name(&mut self, name: &str) -> Result<(), Error> { Ok(self .query(&format!("SET application_name = '{}'", name)) .await?) } + /// Get the servers address. + #[allow(dead_code)] pub fn address(&self) -> Address { self.address.clone() } } impl Drop for Server { - // Try to do a clean shut down. + /// Try to do a clean shut down. Best effort because + /// the socket is in non-blocking mode, so it may not be ready + /// for a write. fn drop(&mut self) { let mut bytes = BytesMut::with_capacity(4); bytes.put_u8(b'X'); bytes.put_i32(4); match self.write.try_write(&bytes) { - Ok(n) => (), + Ok(_) => (), Err(_) => (), }; diff --git a/src/sharding.rs b/src/sharding.rs index 1bbb6d879..7aed4d306 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -18,7 +18,7 @@ impl Sharder { let mut lohalf = key as u32; let hihalf = (key >> 32) as u32; lohalf ^= if key >= 0 { hihalf } else { !hihalf }; - Self::combine(0, Self::pg_u32_hash(lohalf)) as usize % self.shards as usize + Self::combine(0, Self::pg_u32_hash(lohalf)) as usize % self.shards } #[inline] From ccb432b8a0d857b7f10c0fbda3216d6352d129bc Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 16 Feb 2022 22:52:11 -0800 Subject: [PATCH 088/551] Refactor query routing into its own module (#22) * Refactor query routing into its own module * commments; tests; dead code * error message * safer startup * hm * dont have to be public * wow * fix ci * ok * nl * no more silent errors --- src/client.rs | 148 ++-------- src/main.rs | 13 +- src/messages.rs | 44 +++ src/pool.rs | 16 +- src/query_router.rs | 259 ++++++++++++++++++ tests/sharding/partition_hash_test_setup.sql | 2 +- tests/sharding/query_routing_setup.sql | 2 +- tests/sharding/query_routing_test_insert.sql | 10 +- .../query_routing_test_primary_replica.sql | 8 + tests/sharding/query_routing_test_select.sql | 4 +- .../sharding/query_routing_test_validate.sql | 2 +- 11 files changed, 364 insertions(+), 144 deletions(-) create mode 100644 src/query_router.rs diff --git a/src/client.rs b/src/client.rs index 0f3fd3179..dd1b7cca1 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,8 +2,6 @@ /// We are pretending to the server in this scenario, /// and this module implements that. use bytes::{Buf, BufMut, BytesMut}; -use once_cell::sync::OnceCell; -use regex::Regex; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, @@ -17,16 +15,10 @@ use crate::constants::*; use crate::errors::Error; use crate::messages::*; use crate::pool::{ClientServerMap, ConnectionPool}; +use crate::query_router::QueryRouter; use crate::server::Server; -use crate::sharding::Sharder; use crate::stats::Reporter; -pub const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+';"; -pub const ROLE_REGEX: &str = r"SET SERVER ROLE TO '(PRIMARY|REPLICA)';"; - -pub static SHARDING_REGEX_RE: OnceCell = OnceCell::new(); -pub static ROLE_REGEX_RE: OnceCell = OnceCell::new(); - /// The client state. One of these is created per client. pub struct Client { // The reads are buffered (8K by default). @@ -199,15 +191,11 @@ impl Client { return Ok(Server::cancel(&address, &port, process_id, secret_key).await?); } - // Active shard we're talking to. - // The lifetime of this depends on the pool mode: - // - if in session mode, this lives until the client disconnects, - // - if in transaction mode, this lives for the duration of one transaction. - let mut shard: Option = None; - - // Active database role we want to talk to, e.g. primary or replica. - let mut role: Option = self.default_server_role; + let mut query_router = QueryRouter::new(self.default_server_role, pool.shards()); + // Our custom protocol loop. + // We expect the client to either start a transaction with regular queries + // or issue commands for our sharding and server selection protocols. loop { // Read a complete message from the client, which normally would be // either a `Q` (query) or `P` (prepare, extended protocol). @@ -218,32 +206,31 @@ impl Client { // Parse for special select shard command. // SET SHARDING KEY TO 'bigint'; - match self.select_shard(message.clone(), pool.shards()) { - Some(s) => { - custom_protocol_response_ok(&mut self.write, "SET SHARDING KEY").await?; - shard = Some(s); - continue; - } - None => (), - }; + if query_router.select_shard(message.clone()) { + custom_protocol_response_ok( + &mut self.write, + &format!("SET SHARD TO {}", query_router.shard()), + ) + .await?; + continue; + } // Parse for special server role selection command. // SET SERVER ROLE TO '(primary|replica)'; - match self.select_role(message.clone()) { - Some(r) => { - custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; - role = Some(r); - continue; - } - None => (), - }; + if query_router.select_role(message.clone()) { + custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; + continue; + } - // Grab a server from the pool. - let connection = match pool.get(shard, role).await { + // Grab a server from the pool: the client issued a regular query. + let connection = match pool.get(query_router.shard(), query_router.role()).await { Ok(conn) => conn, Err(err) => { println!(">> Could not get connection from pool: {:?}", err); - return Err(err); + error_response(&mut self.write, "could not get connection from the pool") + .await?; + query_router.reset(); + continue; } }; @@ -264,11 +251,8 @@ impl Client { Err(err) => { // Client disconnected without warning. if server.in_transaction() { - // TODO: this is what PgBouncer does - // which leads to connection thrashing. - // - // I think we could issue a ROLLBACK here instead. - // server.mark_bad(); + // Client left dirty server. Clean up and proceed + // without thrashing this connection. server.query("ROLLBACK; DISCARD ALL;").await?; } @@ -328,8 +312,7 @@ impl Client { // Report this client as idle. self.stats.client_idle(); - shard = None; - role = self.default_server_role; + query_router.reset(); break; } @@ -414,8 +397,7 @@ impl Client { if self.transaction_mode { self.stats.client_idle(); - shard = None; - role = self.default_server_role; + query_router.reset(); break; } @@ -450,8 +432,7 @@ impl Client { self.stats.transaction(); if self.transaction_mode { - shard = None; - role = self.default_server_role; + query_router.reset(); break; } @@ -476,77 +457,4 @@ impl Client { let mut guard = self.client_server_map.lock().unwrap(); guard.remove(&(self.process_id, self.secret_key)); } - - /// Determine if the query is part of our special syntax, extract - /// the shard key, and return the shard to query based on Postgres' - /// PARTITION BY HASH function. - fn select_shard(&self, mut buf: BytesMut, shards: usize) -> Option { - let code = buf.get_u8() as char; - - // Only supporting simpe protocol here, so - // one would have to execute something like this: - // psql -c "SET SHARDING KEY TO '1234'" - // after sanitizing the value manually, which can be just done with an - // int parser, e.g. `let key = "1234".parse::().unwrap()`. - match code { - 'Q' => (), - _ => return None, - }; - - let len = buf.get_i32(); - let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); // Don't read the ternminating null - - let rgx = match SHARDING_REGEX_RE.get() { - Some(r) => r, - None => return None, - }; - - if rgx.is_match(&query) { - let shard = query.split("'").collect::>()[1]; - - match shard.parse::() { - Ok(shard) => { - let sharder = Sharder::new(shards); - Some(sharder.pg_bigint_hash(shard)) - } - - Err(_) => None, - } - } else { - None - } - } - - // Pick a primary or a replica from the pool. - fn select_role(&self, mut buf: BytesMut) -> Option { - let code = buf.get_u8() as char; - - // Same story as select_shard() above. - match code { - 'Q' => (), - _ => return None, - }; - - let len = buf.get_i32(); - let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); - - let rgx = match ROLE_REGEX_RE.get() { - Some(r) => r, - None => return None, - }; - - // Copy / paste from above. If we get one more of these use cases, - // it'll be time to abstract :). - if rgx.is_match(&query) { - let role = query.split("'").collect::>()[1]; - - match role { - "PRIMARY" => Some(Role::Primary), - "REPLICA" => Some(Role::Replica), - _ => return None, - } - } else { - None - } - } } diff --git a/src/main.rs b/src/main.rs index 324f9015e..cd7416218 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,7 +25,6 @@ extern crate statsd; extern crate tokio; extern crate toml; -use regex::Regex; use tokio::net::TcpListener; use tokio::signal; @@ -39,6 +38,7 @@ mod constants; mod errors; mod messages; mod pool; +mod query_router; mod server; mod sharding; mod stats; @@ -54,12 +54,11 @@ use stats::{Collector, Reporter}; async fn main() { println!("> Welcome to PgCat! Meow."); - client::SHARDING_REGEX_RE - .set(Regex::new(client::SHARDING_REGEX).unwrap()) - .unwrap(); - client::ROLE_REGEX_RE - .set(Regex::new(client::ROLE_REGEX).unwrap()) - .unwrap(); + // Prepare regexes + if !query_router::QueryRouter::setup() { + println!("> Could not setup query router."); + return; + } let config = match config::parse("pgcat.toml").await { Ok(config) => config, diff --git a/src/messages.rs b/src/messages.rs index ec5ac66f7..3b5914d06 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -185,6 +185,50 @@ pub async fn custom_protocol_response_ok( write_all_half(stream, res).await } +/// Send a custom error message to the client. +/// Tell the client we are ready for the next query and no rollback is necessary. +/// Docs on error codes: https://www.postgresql.org/docs/12/errcodes-appendix.html +pub async fn error_response(stream: &mut OwnedWriteHalf, message: &str) -> Result<(), Error> { + let mut error = BytesMut::new(); + + // Error level + error.put_u8(b'S'); + error.put_slice(&b"FATAL\0"[..]); + + // Error level (non-translatable) + error.put_u8(b'V'); + error.put_slice(&b"FATAL\0"[..]); + + // Error code: not sure how much this matters. + error.put_u8(b'C'); + error.put_slice(&b"58000\0"[..]); // system_error, see Appendix A. + + // The short error message. + error.put_u8(b'M'); + error.put_slice(&format!("{}\0", message).as_bytes()); + + // No more fields follow. + error.put_u8(0); + + // Ready for query, no rollback needed (I = idle). + let mut ready_for_query = BytesMut::new(); + + ready_for_query.put_u8(b'Z'); + ready_for_query.put_i32(5); + ready_for_query.put_u8(b'I'); + + // Compose the two message reply. + let mut res = BytesMut::with_capacity(error.len() + ready_for_query.len() + 5); + + res.put_u8(b'E'); + res.put_i32(error.len() as i32 + 4); + + res.put(error); + res.put(ready_for_query); + + Ok(write_all_half(stream, res).await?) +} + /// Write all data in the buffer to the TcpStream. pub async fn write_all(stream: &mut TcpStream, buf: BytesMut) -> Result<(), Error> { match stream.write_all(&buf).await { diff --git a/src/pool.rs b/src/pool.rs index d348eec49..a5e32d10d 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -121,7 +121,7 @@ impl ConnectionPool { for shard in 0..self.shards() { for _ in 0..self.replicas(shard) { - let connection = match self.get(Some(shard), None).await { + let connection = match self.get(shard, None).await { Ok(conn) => conn, Err(err) => { println!("> Shard {} down or misconfigured.", shard); @@ -149,18 +149,13 @@ impl ConnectionPool { /// Get a connection from the pool. pub async fn get( &mut self, - shard: Option, + shard: usize, role: Option, ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { // Set this to false to gain ~3-4% speed. let with_health_check = true; let now = Instant::now(); - let shard = match shard { - Some(shard) => shard, - None => 0, // TODO: pick a shard at random - }; - // We are waiting for a server now. self.stats.client_waiting(); @@ -208,11 +203,8 @@ impl ConnectionPool { // as per request. match role { Some(role) => { - // If the client wants a specific role, - // we'll do our best to pick it, but if we only - // have one server in the cluster, it's probably only a primary - // (or only a replica), so the client will just get what we have. - if address.role != role && addresses.len() > 1 { + // Find the specific role the client wants in the pool. + if address.role != role { continue; } } diff --git a/src/query_router.rs b/src/query_router.rs new file mode 100644 index 000000000..aa73d564b --- /dev/null +++ b/src/query_router.rs @@ -0,0 +1,259 @@ +use bytes::{Buf, BytesMut}; +/// Route queries automatically based on explicitely requested +/// or implied query characteristics. +use once_cell::sync::OnceCell; +use regex::{Regex, RegexBuilder}; + +use crate::config::Role; +use crate::sharding::Sharder; + +const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+';"; +const ROLE_REGEX: &str = r"SET SERVER ROLE TO '(PRIMARY|REPLICA)';"; + +static SHARDING_REGEX_RE: OnceCell = OnceCell::new(); +static ROLE_REGEX_RE: OnceCell = OnceCell::new(); + +pub struct QueryRouter { + // By default, queries go here, unless we have better information + // about what the client wants. + default_server_role: Option, + + // Number of shards in the cluster. + shards: usize, + + // Which shard we should be talking to right now. + active_shard: Option, + + // Should we be talking to a primary or a replica? + active_role: Option, +} + +impl QueryRouter { + pub fn setup() -> bool { + // Compile our query routing regexes early, so we only do it once. + let a = match SHARDING_REGEX_RE.set( + RegexBuilder::new(SHARDING_REGEX) + .case_insensitive(true) + .build() + .unwrap(), + ) { + Ok(_) => true, + Err(_) => false, + }; + + let b = match ROLE_REGEX_RE.set( + RegexBuilder::new(ROLE_REGEX) + .case_insensitive(true) + .build() + .unwrap(), + ) { + Ok(_) => true, + Err(_) => false, + }; + + a && b + } + + pub fn new(default_server_role: Option, shards: usize) -> QueryRouter { + QueryRouter { + default_server_role: default_server_role, + shards: shards, + + active_role: default_server_role, + active_shard: None, + } + } + + /// Determine if the query is part of our special syntax, extract + /// the shard key, and return the shard to query based on Postgres' + /// PARTITION BY HASH function. + pub fn select_shard(&mut self, mut buf: BytesMut) -> bool { + let code = buf.get_u8() as char; + + // Only supporting simpe protocol here, so + // one would have to execute something like this: + // psql -c "SET SHARDING KEY TO '1234'" + // after sanitizing the value manually, which can be just done with an + // int parser, e.g. `let key = "1234".parse::().unwrap()`. + match code { + 'Q' => (), + _ => return false, + }; + + let len = buf.get_i32(); + let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]); // Don't read the ternminating null + + let rgx = match SHARDING_REGEX_RE.get() { + Some(r) => r, + None => return false, + }; + + if rgx.is_match(&query) { + let shard = query.split("'").collect::>()[1]; + + match shard.parse::() { + Ok(shard) => { + let sharder = Sharder::new(self.shards); + self.active_shard = Some(sharder.pg_bigint_hash(shard)); + + true + } + + // The shard must be a valid integer. Our regex won't let anything else pass, + // so this code will never run, but Rust can't know that, so we have to handle this + // case anyway. + Err(_) => false, + } + } else { + false + } + } + + // Pick a primary or a replica from the pool. + pub fn select_role(&mut self, mut buf: BytesMut) -> bool { + let code = buf.get_u8() as char; + + // Same story as select_shard() above. + match code { + 'Q' => (), + _ => return false, + }; + + let len = buf.get_i32(); + let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); + + let rgx = match ROLE_REGEX_RE.get() { + Some(r) => r, + None => return false, + }; + + // Copy / paste from above. If we get one more of these use cases, + // it'll be time to abstract :). + if rgx.is_match(&query) { + let role = query.split("'").collect::>()[1]; + + match role { + "PRIMARY" => { + self.active_role = Some(Role::Primary); + true + } + "REPLICA" => { + self.active_role = Some(Role::Replica); + true + } + + // Our regex won't let this case happen, but Rust can't know that. + _ => false, + } + } else { + false + } + } + + /// Get the current desired server role we should be talking to. + pub fn role(&self) -> Option { + self.active_role + } + + /// Get desired shard we should be talking to. + pub fn shard(&self) -> usize { + match self.active_shard { + Some(shard) => shard, + None => 0, // TODO: pick random shard + } + } + + /// Reset the router back to defaults. + /// This must be called at the end of every transaction in transaction mode. + pub fn reset(&mut self) { + self.active_role = self.default_server_role; + self.active_shard = None; + } +} + +#[cfg(test)] +mod test { + use super::*; + use bytes::BufMut; + + #[test] + fn test_select_shard() { + QueryRouter::setup(); + + let default_server_role: Option = None; + let shards = 5; + let mut query_router = QueryRouter::new(default_server_role, shards); + + // Build the special syntax query. + let mut message = BytesMut::new(); + let query = BytesMut::from(&b"SET SHARDING KEY TO '13';\0"[..]); + + message.put_u8(b'Q'); // Query + message.put_i32(query.len() as i32 + 4); + message.put_slice(&query[..]); + + assert!(query_router.select_shard(message)); + assert_eq!(query_router.shard(), 3); // See sharding.rs (we are using 5 shards on purpose in this test) + + query_router.reset(); + assert_eq!(query_router.shard(), 0); + } + + #[test] + fn test_select_replica() { + QueryRouter::setup(); + + let default_server_role: Option = None; + let shards = 5; + let mut query_router = QueryRouter::new(default_server_role, shards); + + // Build the special syntax query. + let mut message = BytesMut::new(); + let query = BytesMut::from(&b"SET SERVER ROLE TO 'replica';\0"[..]); + + message.put_u8(b'Q'); // Query + message.put_i32(query.len() as i32 + 4); + message.put_slice(&query[..]); + + assert!(query_router.select_role(message)); + assert_eq!(query_router.role(), Some(Role::Replica)); + + query_router.reset(); + + assert_eq!(query_router.role(), default_server_role); + } + + #[test] + fn test_defaults() { + QueryRouter::setup(); + + let default_server_role: Option = None; + let shards = 5; + let query_router = QueryRouter::new(default_server_role, shards); + + assert_eq!(query_router.shard(), 0); + assert_eq!(query_router.role(), None); + } + + #[test] + fn test_incorrect_syntax() { + QueryRouter::setup(); + + let default_server_role: Option = None; + let shards = 5; + let mut query_router = QueryRouter::new(default_server_role, shards); + + // Build the special syntax query. + let mut message = BytesMut::new(); + + // Typo! + let query = BytesMut::from(&b"SET SERVER RLE TO 'replica';\0"[..]); + + message.put_u8(b'Q'); // Query + message.put_i32(query.len() as i32 + 4); + message.put_slice(&query[..]); + + assert_eq!(query_router.select_shard(message.clone()), false); + assert_eq!(query_router.select_role(message.clone()), false); + } +} diff --git a/tests/sharding/partition_hash_test_setup.sql b/tests/sharding/partition_hash_test_setup.sql index e802ead23..5e9102061 100644 --- a/tests/sharding/partition_hash_test_setup.sql +++ b/tests/sharding/partition_hash_test_setup.sql @@ -23,4 +23,4 @@ SELECT * FROM shard_0 ORDER BY id LIMIT 10; SELECT * FROM shard_1 ORDER BY id LIMIT 10; SELECT * FROM shard_2 ORDER BY id LIMIT 10; SELECT * FROM shard_3 ORDER BY id LIMIT 10; -SELECT * FROM shard_4 ORDER BY id LIMIT 10; \ No newline at end of file +SELECT * FROM shard_4 ORDER BY id LIMIT 10; diff --git a/tests/sharding/query_routing_setup.sql b/tests/sharding/query_routing_setup.sql index d4e766dab..8ecfc58a8 100644 --- a/tests/sharding/query_routing_setup.sql +++ b/tests/sharding/query_routing_setup.sql @@ -58,4 +58,4 @@ GRANT ALL ON TABLE data TO sharding_user; \c shard2 GRANT ALL ON SCHEMA public TO sharding_user; -GRANT ALL ON TABLE data TO sharding_user; \ No newline at end of file +GRANT ALL ON TABLE data TO sharding_user; diff --git a/tests/sharding/query_routing_test_insert.sql b/tests/sharding/query_routing_test_insert.sql index 97d5babd6..ff8a32611 100644 --- a/tests/sharding/query_routing_test_insert.sql +++ b/tests/sharding/query_routing_test_insert.sql @@ -1,3 +1,5 @@ +\set ON_ERROR_STOP on + SET SHARDING KEY TO '1'; INSERT INTO data (id, value) VALUES (1, 'value_1'); @@ -44,4 +46,10 @@ SET SHARDING KEY TO '15'; INSERT INTO data (id, value) VALUES (15, 'value_1'); SET SHARDING KEY TO '16'; -INSERT INTO data (id, value) VALUES (16, 'value_1'); \ No newline at end of file +INSERT INTO data (id, value) VALUES (16, 'value_1'); + +set sharding key to '17'; +INSERT INTO data (id, value) VALUES (17, 'value_1'); + +SeT SHaRDInG KeY to '18'; +INSERT INTO data (id, value) VALUES (18, 'value_1'); diff --git a/tests/sharding/query_routing_test_primary_replica.sql b/tests/sharding/query_routing_test_primary_replica.sql index 358b073c6..db05ba7a8 100644 --- a/tests/sharding/query_routing_test_primary_replica.sql +++ b/tests/sharding/query_routing_test_primary_replica.sql @@ -1,3 +1,5 @@ +\set ON_ERROR_STOP on + SET SERVER ROLE TO 'primary'; SET SHARDING KEY TO '1'; INSERT INTO data (id, value) VALUES (1, 'value_1'); @@ -88,6 +90,8 @@ SELECT * FROM data WHERE id = 9; --- +\set ON_ERROR_STOP on + SET SERVER ROLE TO 'primary'; SET SHARDING KEY TO '10'; INSERT INTO data (id, value) VALUES (10, 'value_1'); @@ -143,3 +147,7 @@ SELECT 1; SET SERVER ROLE TO 'replica'; SELECT 1; + +set server role to 'replica'; +SeT SeRver Role TO 'PrImARY'; +select 1; diff --git a/tests/sharding/query_routing_test_select.sql b/tests/sharding/query_routing_test_select.sql index c5778034e..1b30fdf85 100644 --- a/tests/sharding/query_routing_test_select.sql +++ b/tests/sharding/query_routing_test_select.sql @@ -1,3 +1,5 @@ +\set ON_ERROR_STOP on + SET SHARDING KEY TO '1'; SELECT * FROM data WHERE id = 1; @@ -44,4 +46,4 @@ SET SHARDING KEY TO '15'; SELECT * FROM data WHERE id = 15; SET SHARDING KEY TO '16'; -SELECT * FROM data WHERE id = 16; \ No newline at end of file +SELECT * FROM data WHERE id = 16; diff --git a/tests/sharding/query_routing_test_validate.sql b/tests/sharding/query_routing_test_validate.sql index 5ef9a5686..1647087c9 100644 --- a/tests/sharding/query_routing_test_validate.sql +++ b/tests/sharding/query_routing_test_validate.sql @@ -8,4 +8,4 @@ SELECT * FROM data; \c shard2 -SELECT * FROM data; \ No newline at end of file +SELECT * FROM data; From e627978636b08ebe1663e4cefe457ee33f4b7fdb Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 18 Feb 2022 07:10:18 -0800 Subject: [PATCH 089/551] Query parser 3.0 (#23) * Starting query parsing * Query parser * working config * disable by default * fix tsets * introducing log crate; test for query router; comments * typo * fixes for banning * added test for prepared stmt --- Cargo.lock | 11 +++ Cargo.toml | 2 + pgcat.toml | 12 +++ src/client.rs | 21 +++-- src/config.rs | 20 +++++ src/main.rs | 15 +++- src/pool.rs | 93 +++++++++------------ src/query_router.rs | 197 ++++++++++++++++++++++++++++++++++++++++++-- 8 files changed, 296 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1673d3a77..d11410a32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -322,6 +322,7 @@ dependencies = [ "bb8", "bytes", "chrono", + "log", "md-5", "num_cpus", "once_cell", @@ -330,6 +331,7 @@ dependencies = [ "serde", "serde_derive", "sha-1", + "sqlparser", "statsd", "tokio", "toml", @@ -492,6 +494,15 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "sqlparser" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8f192f29f4aa49e57bebd0aa05858e0a1f32dd270af36efe49edb82cbfffab6" +dependencies = [ + "log", +] + [[package]] name = "statsd" version = "0.15.0" diff --git a/Cargo.toml b/Cargo.toml index 0db9ecdd0..ba78b59bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,3 +21,5 @@ regex = "1" num_cpus = "1" once_cell = "1" statsd = "0.15" +sqlparser = "0.14" +log = "0.4" diff --git a/pgcat.toml b/pgcat.toml index db5c3f2f5..0fa8b6a0a 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -81,3 +81,15 @@ database = "shard2" # replica: round-robin between replicas only without touching the primary, # primary: all queries go to the primary unless otherwise specified. default_role = "any" + + +# Query parser. If enabled, we'll attempt to parse +# every incoming query to determine if it's a read or a write. +# If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, +# we'll direct it to the primary. +query_parser_enabled = false + +# If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for +# load balancing of read queries. Otherwise, the primary will only be used for write +# queries. The primary can always be explicitely selected with our custom protocol. +primary_reads_enabled = true diff --git a/src/client.rs b/src/client.rs index dd1b7cca1..184d46d9f 100644 --- a/src/client.rs +++ b/src/client.rs @@ -10,7 +10,6 @@ use tokio::net::{ use std::collections::HashMap; -use crate::config::Role; use crate::constants::*; use crate::errors::Error; use crate::messages::*; @@ -47,10 +46,6 @@ pub struct Client { // to connect and cancel a query. client_server_map: ClientServerMap, - // Unless client specifies, route queries to the servers that have this role, - // e.g. primary or replicas or any. - default_server_role: Option, - // Client parameters, e.g. user, client_encoding, etc. #[allow(dead_code)] parameters: HashMap, @@ -67,7 +62,6 @@ impl Client { mut stream: TcpStream, client_server_map: ClientServerMap, transaction_mode: bool, - default_server_role: Option, server_info: BytesMut, stats: Reporter, ) -> Result { @@ -126,7 +120,6 @@ impl Client { process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, - default_server_role: default_server_role, parameters: parameters, stats: stats, }); @@ -148,7 +141,6 @@ impl Client { process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, - default_server_role: default_server_role, parameters: HashMap::new(), stats: stats, }); @@ -162,7 +154,11 @@ impl Client { } /// Client loop. We handle all messages between the client and the database here. - pub async fn handle(&mut self, mut pool: ConnectionPool) -> Result<(), Error> { + pub async fn handle( + &mut self, + mut pool: ConnectionPool, + mut query_router: QueryRouter, + ) -> Result<(), Error> { // The client wants to cancel a query it has issued previously. if self.cancel_mode { let (process_id, secret_key, address, port) = { @@ -191,8 +187,6 @@ impl Client { return Ok(Server::cancel(&address, &port, process_id, secret_key).await?); } - let mut query_router = QueryRouter::new(self.default_server_role, pool.shards()); - // Our custom protocol loop. // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocols. @@ -222,6 +216,11 @@ impl Client { continue; } + // Attempt to parse the query to determine where it should go + if query_router.query_parser_enabled() && query_router.role() == None { + query_router.infer_role(message.clone()); + } + // Grab a server from the pool: the client issued a regular query. let connection = match pool.get(query_router.shard(), query_router.role()).await { Ok(conn) => conn, diff --git a/src/config.rs b/src/config.rs index 1a3f22b50..8f7c45fa2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -13,6 +13,24 @@ pub enum Role { Replica, } +impl PartialEq> for Role { + fn eq(&self, other: &Option) -> bool { + match other { + None => true, + Some(role) => *self == *role, + } + } +} + +impl PartialEq for Option { + fn eq(&self, other: &Role) -> bool { + match *self { + None => true, + Some(role) => role == *other, + } + } +} + #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)] pub struct Address { pub host: String, @@ -47,6 +65,8 @@ pub struct Shard { #[derive(Deserialize, Debug, Clone)] pub struct QueryRouter { pub default_role: String, + pub query_parser_enabled: bool, + pub primary_reads_enabled: bool, } #[derive(Deserialize, Debug, Clone)] diff --git a/src/main.rs b/src/main.rs index cd7416218..1669331ad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,11 +16,13 @@ extern crate async_trait; extern crate bb8; extern crate bytes; +extern crate log; extern crate md5; extern crate num_cpus; extern crate once_cell; extern crate serde; extern crate serde_derive; +extern crate sqlparser; extern crate statsd; extern crate tokio; extern crate toml; @@ -47,6 +49,7 @@ mod stats; // secret keys to the backend's. use config::Role; use pool::{ClientServerMap, ConnectionPool}; +use query_router::QueryRouter; use stats::{Collector, Reporter}; /// Main! @@ -118,6 +121,8 @@ async fn main() { return; } }; + let primary_reads_enabled = config.query_router.primary_reads_enabled; + let query_parser_enabled = config.query_router.query_parser_enabled; let server_info = match pool.validate().await { Ok(info) => info, @@ -155,7 +160,6 @@ async fn main() { socket, client_server_map, transaction_mode, - default_server_role, server_info, reporter, ) @@ -164,7 +168,14 @@ async fn main() { Ok(mut client) => { println!(">> Client {:?} authenticated successfully!", addr); - match client.handle(pool).await { + let query_router = QueryRouter::new( + default_server_role, + pool.shards(), + primary_reads_enabled, + query_parser_enabled, + ); + + match client.handle(pool, query_router).await { Ok(()) => { let duration = chrono::offset::Utc::now().naive_utc() - start; diff --git a/src/pool.rs b/src/pool.rs index a5e32d10d..6a190f11e 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -25,7 +25,6 @@ pub struct ConnectionPool { banlist: BanList, healthcheck_timeout: u64, ban_time: i64, - pool_size: u32, stats: Reporter, } @@ -47,12 +46,12 @@ impl ConnectionPool { .collect::>(); shard_ids.sort_by_key(|k| k.parse::().unwrap()); - for shard in shard_ids { - let shard = &config.shards[&shard]; + for shard_idx in shard_ids { + let shard = &config.shards[&shard_idx]; let mut pools = Vec::new(); let mut replica_addresses = Vec::new(); - for (idx, server) in shard.servers.iter().enumerate() { + for server in shard.servers.iter() { let role = match server.2.as_ref() { "primary" => Role::Primary, "replica" => Role::Replica, @@ -66,7 +65,7 @@ impl ConnectionPool { host: server.0.clone(), port: server.1.to_string(), role: role, - shard: idx, + shard: shard_idx.parse::().unwrap(), }; let manager = ServerPool::new( @@ -106,7 +105,6 @@ impl ConnectionPool { banlist: Arc::new(Mutex::new(banlist)), healthcheck_timeout: config.general.healthcheck_timeout, ban_time: config.general.ban_time, - pool_size: config.general.pool_size, stats: stats, } } @@ -120,12 +118,12 @@ impl ConnectionPool { let mut server_infos = Vec::new(); for shard in 0..self.shards() { - for _ in 0..self.replicas(shard) { + for _ in 0..self.servers(shard) { let connection = match self.get(shard, None).await { Ok(conn) => conn, Err(err) => { - println!("> Shard {} down or misconfigured.", shard); - return Err(err); + println!("> Shard {} down or misconfigured: {:?}", shard, err); + continue; } }; @@ -152,8 +150,6 @@ impl ConnectionPool { shard: usize, role: Option, ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { - // Set this to false to gain ~3-4% speed. - let with_health_check = true; let now = Instant::now(); // We are waiting for a server now. @@ -161,25 +157,6 @@ impl ConnectionPool { let addresses = &self.addresses[shard]; - // Make sure if a specific role is requested, it's available in the pool. - match role { - Some(role) => { - let role_count = addresses.iter().filter(|&db| db.role == role).count(); - - if role_count == 0 { - println!( - ">> Error: Role '{:?}' requested, but none are configured.", - role - ); - - return Err(Error::AllServersDown); - } - } - - // Any role should be present. - _ => (), - }; - let mut allowed_attempts = match role { // Primary-specific queries get one attempt, if the primary is down, // nothing we should do about it I think. It's dangerous to retry @@ -188,9 +165,22 @@ impl ConnectionPool { // Replicas get to try as many times as there are replicas // and connections in the pool. - _ => self.databases[shard].len() * self.pool_size as usize, + _ => addresses.len(), + }; + + let exists = match role { + Some(role) => addresses.iter().filter(|addr| addr.role == role).count() > 0, + None => true, }; + if !exists { + log::error!( + "ConnectionPool::get Requested role {:?}, but none is configured.", + role + ); + return Err(Error::BadConfig); + } + while allowed_attempts > 0 { // Round-robin each client's queries. // If a client only sends one query and then disconnects, it doesn't matter @@ -200,23 +190,18 @@ impl ConnectionPool { let address = &addresses[index]; // Make sure you're getting a primary or a replica - // as per request. - match role { - Some(role) => { - // Find the specific role the client wants in the pool. - if address.role != role { - continue; - } - } - None => (), - }; - - if self.is_banned(address, shard, role) { + // as per request. If no specific role is requested, the first + // available will be chosen. + if address.role != role { continue; } allowed_attempts -= 1; + if self.is_banned(address, shard, role) { + continue; + } + // Check if we can connect let mut conn = match self.databases[shard][index].get().await { Ok(conn) => conn, @@ -227,12 +212,6 @@ impl ConnectionPool { } }; - if !with_health_check { - self.stats.checkout_time(now.elapsed().as_micros()); - self.stats.client_active(); - return Ok((conn, address.clone())); - } - // // Check if this server is alive with a health check let server = &mut *conn; @@ -299,17 +278,21 @@ impl ConnectionPool { /// Check if a replica can serve traffic. If all replicas are banned, /// we unban all of them. Better to try then not to. pub fn is_banned(&self, address: &Address, shard: usize, role: Option) -> bool { - // If primary is requested explicitely, it can never be banned. - if Some(Role::Primary) == role { - return false; - } + let replicas_available = match role { + Some(Role::Replica) => self.addresses[shard] + .iter() + .filter(|addr| addr.role == Role::Replica) + .count(), + None => self.addresses[shard].len(), + Some(Role::Primary) => return false, // Primary cannot be banned. + }; // If you're not asking for the primary, // all databases are treated as replicas. let mut guard = self.banlist.lock().unwrap(); // Everything is banned = nothing is banned. - if guard[shard].len() == self.databases[shard].len() { + if guard[shard].len() == replicas_available { guard[shard].clear(); drop(guard); println!(">> Unbanning all replicas."); @@ -337,7 +320,7 @@ impl ConnectionPool { self.databases.len() } - pub fn replicas(&self, shard: usize) -> usize { + pub fn servers(&self, shard: usize) -> usize { self.addresses[shard].len() } } diff --git a/src/query_router.rs b/src/query_router.rs index aa73d564b..f5bdc998f 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1,8 +1,11 @@ -use bytes::{Buf, BytesMut}; /// Route queries automatically based on explicitely requested /// or implied query characteristics. +use bytes::{Buf, BytesMut}; use once_cell::sync::OnceCell; use regex::{Regex, RegexBuilder}; +use sqlparser::ast::Statement::{Query, StartTransaction}; +use sqlparser::dialect::PostgreSqlDialect; +use sqlparser::parser::Parser; use crate::config::Role; use crate::sharding::Sharder; @@ -26,6 +29,12 @@ pub struct QueryRouter { // Should we be talking to a primary or a replica? active_role: Option, + + // Include the primary into the replica pool? + primary_reads_enabled: bool, + + // Should we try to parse queries? + query_parser_enabled: bool, } impl QueryRouter { @@ -54,13 +63,20 @@ impl QueryRouter { a && b } - pub fn new(default_server_role: Option, shards: usize) -> QueryRouter { + pub fn new( + default_server_role: Option, + shards: usize, + primary_reads_enabled: bool, + query_parser_enabled: bool, + ) -> QueryRouter { QueryRouter { default_server_role: default_server_role, shards: shards, active_role: default_server_role, active_shard: None, + primary_reads_enabled: primary_reads_enabled, + query_parser_enabled: query_parser_enabled, } } @@ -109,7 +125,7 @@ impl QueryRouter { } } - // Pick a primary or a replica from the pool. + /// Pick a primary or a replica from the pool. pub fn select_role(&mut self, mut buf: BytesMut) -> bool { let code = buf.get_u8() as char; @@ -150,6 +166,75 @@ impl QueryRouter { } } + /// Try to infer which server to connect to based on the contents of the query. + pub fn infer_role(&mut self, mut buf: BytesMut) -> bool { + let code = buf.get_u8() as char; + let len = buf.get_i32() as usize; + + let query = match code { + 'Q' => String::from_utf8_lossy(&buf[..len - 5]).to_string(), + 'P' => { + let mut start = 0; + let mut end; + + // Skip the name of the prepared statement. + while buf[start] != 0 && start < buf.len() { + start += 1; + } + start += 1; // Skip terminating null + + // Find the end of the prepared stmt (\0) + end = start; + while buf[end] != 0 && end < buf.len() { + end += 1; + } + + let query = String::from_utf8_lossy(&buf[start..end]).to_string(); + + query.replace("$", "") // Remove placeholders turning them into "values" + } + _ => return false, + }; + + let ast = match Parser::parse_sql(&PostgreSqlDialect {}, &query) { + Ok(ast) => ast, + Err(err) => { + log::debug!( + "QueryParser::infer_role could not parse query, error: {:?}, query: {}", + err, + query + ); + return false; + } + }; + + if ast.len() == 0 { + return false; + } + + match ast[0] { + // All transactions go to the primary, probably a write. + StartTransaction { .. } => { + self.active_role = Some(Role::Primary); + } + + // Likely a read-only query + Query { .. } => { + self.active_role = match self.primary_reads_enabled { + false => Some(Role::Replica), // If primary should not be receiving reads, use a replica. + true => None, // Any server role is fine in this case. + } + } + + // Likely a write + _ => { + self.active_role = Some(Role::Primary); + } + }; + + true + } + /// Get the current desired server role we should be talking to. pub fn role(&self) -> Option { self.active_role @@ -169,6 +254,11 @@ impl QueryRouter { self.active_role = self.default_server_role; self.active_shard = None; } + + /// Should we attempt to parse queries? + pub fn query_parser_enabled(&self) -> bool { + self.query_parser_enabled + } } #[cfg(test)] @@ -182,7 +272,7 @@ mod test { let default_server_role: Option = None; let shards = 5; - let mut query_router = QueryRouter::new(default_server_role, shards); + let mut query_router = QueryRouter::new(default_server_role, shards, false, false); // Build the special syntax query. let mut message = BytesMut::new(); @@ -205,7 +295,7 @@ mod test { let default_server_role: Option = None; let shards = 5; - let mut query_router = QueryRouter::new(default_server_role, shards); + let mut query_router = QueryRouter::new(default_server_role, shards, false, false); // Build the special syntax query. let mut message = BytesMut::new(); @@ -229,7 +319,7 @@ mod test { let default_server_role: Option = None; let shards = 5; - let query_router = QueryRouter::new(default_server_role, shards); + let query_router = QueryRouter::new(default_server_role, shards, false, false); assert_eq!(query_router.shard(), 0); assert_eq!(query_router.role(), None); @@ -241,7 +331,7 @@ mod test { let default_server_role: Option = None; let shards = 5; - let mut query_router = QueryRouter::new(default_server_role, shards); + let mut query_router = QueryRouter::new(default_server_role, shards, false, false); // Build the special syntax query. let mut message = BytesMut::new(); @@ -256,4 +346,97 @@ mod test { assert_eq!(query_router.select_shard(message.clone()), false); assert_eq!(query_router.select_role(message.clone()), false); } + + #[test] + fn test_infer_role_replica() { + QueryRouter::setup(); + + let default_server_role: Option = None; + let shards = 5; + + let mut query_router = QueryRouter::new(default_server_role, shards, false, false); + + let queries = vec![ + BytesMut::from(&b"SELECT * FROM items WHERE id = 5\0"[..]), + BytesMut::from(&b"SELECT id, name, value FROM items INNER JOIN prices ON item.id = prices.item_id\0"[..]), + BytesMut::from(&b"WITH t AS (SELECT * FROM items) SELECT * FROM t\0"[..]), + ]; + + for query in &queries { + let mut res = BytesMut::from(&b"Q"[..]); + res.put_i32(query.len() as i32 + 4); + res.put(query.clone()); + + // It's a recognized query + assert!(query_router.infer_role(res)); + assert_eq!(query_router.role(), Some(Role::Replica)); + } + } + + #[test] + fn test_infer_role_primary() { + QueryRouter::setup(); + + let default_server_role: Option = None; + let shards = 5; + + let mut query_router = QueryRouter::new(default_server_role, shards, false, false); + + let queries = vec![ + BytesMut::from(&b"UPDATE items SET name = 'pumpkin' WHERE id = 5\0"[..]), + BytesMut::from(&b"INSERT INTO items (id, name) VALUES (5, 'pumpkin')\0"[..]), + BytesMut::from(&b"DELETE FROM items WHERE id = 5\0"[..]), + BytesMut::from(&b"BEGIN\0"[..]), // Transaction start + ]; + + for query in &queries { + let mut res = BytesMut::from(&b"Q"[..]); + res.put_i32(query.len() as i32 + 4); + res.put(query.clone()); + + // It's a recognized query + assert!(query_router.infer_role(res)); + assert_eq!(query_router.role(), Some(Role::Primary)); + } + } + + #[test] + fn test_infer_role_primary_reads_enabled() { + QueryRouter::setup(); + + let default_server_role: Option = None; + let shards = 5; + + let mut query_router = QueryRouter::new(default_server_role, shards, true, false); + + let query = BytesMut::from(&b"SELECT * FROM items WHERE id = 5\0"[..]); + let mut res = BytesMut::from(&b"Q"[..]); + res.put_i32(query.len() as i32 + 4); + res.put(query.clone()); + + assert!(query_router.infer_role(res)); + assert_eq!(query_router.role(), None); + } + + #[test] + fn test_infer_role_parse_prepared() { + QueryRouter::setup(); + + let default_server_role: Option = None; + let shards = 5; + + let mut query_router = QueryRouter::new(default_server_role, shards, false, false); + + let prepared_stmt = BytesMut::from( + &b"WITH t AS (SELECT * FROM items WHERE name = $1) SELECT * FROM t WHERE id = $2\0"[..], + ); + let mut res = BytesMut::from(&b"P"[..]); + res.put_i32(prepared_stmt.len() as i32 + 4 + 1 + 2); + res.put_u8(0); + res.put(prepared_stmt); + res.put_i16(0); + + assert!(query_router.infer_role(res)); + assert_eq!(query_router.role(), Some(Role::Replica)); + } } From 7b5fcf224a2634477b83d39b58c0a52da25e1d11 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 18 Feb 2022 09:43:07 -0800 Subject: [PATCH 090/551] Explicit shard selection; Rails tests (#24) * Explicit shard selection; Rails tests * try running ruby tests * try without lockfile * aha * ok --- .circleci/config.yml | 2 +- .circleci/run_tests.sh | 8 ++++ src/query_router.rs | 60 +++++++++++++++++++++++++++--- tests/ruby/.ruby-version | 1 + tests/ruby/Gemfile | 4 ++ tests/ruby/Gemfile.lock | 30 +++++++++++++++ tests/ruby/tests.rb | 79 ++++++++++++++++++++++++++++++++++++---- 7 files changed, 170 insertions(+), 14 deletions(-) create mode 100644 tests/ruby/.ruby-version create mode 100644 tests/ruby/Gemfile create mode 100644 tests/ruby/Gemfile.lock diff --git a/.circleci/config.yml b/.circleci/config.yml index 5feb5c583..f6180ad5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: key: cargo-lock-2-{{ checksum "Cargo.lock" }} - run: name: "Install dependencies" - command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12" + command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev" - run: name: "Build" command: "cargo build" diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 2ff300827..8a7310281 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -34,6 +34,14 @@ psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql > / # Replica/primary selection & more sharding tests psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null +# +# ActiveRecord tests! +# +cd tests/ruby +sudo gem install bundler +bundle install +ruby tests.rb + # Attempt clean shut down killall pgcat -s SIGINT diff --git a/src/query_router.rs b/src/query_router.rs index f5bdc998f..c02e6a615 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -10,11 +10,13 @@ use sqlparser::parser::Parser; use crate::config::Role; use crate::sharding::Sharder; -const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+';"; -const ROLE_REGEX: &str = r"SET SERVER ROLE TO '(PRIMARY|REPLICA)';"; +const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+'"; +const SET_SHARD_REGEX: &str = r"SET SHARD TO '[0-9]+'"; +const ROLE_REGEX: &str = r"SET SERVER ROLE TO '(PRIMARY|REPLICA)'"; static SHARDING_REGEX_RE: OnceCell = OnceCell::new(); static ROLE_REGEX_RE: OnceCell = OnceCell::new(); +static SET_SHARD_REGEX_RE: OnceCell = OnceCell::new(); pub struct QueryRouter { // By default, queries go here, unless we have better information @@ -60,7 +62,17 @@ impl QueryRouter { Err(_) => false, }; - a && b + let c = match SET_SHARD_REGEX_RE.set( + RegexBuilder::new(SET_SHARD_REGEX) + .case_insensitive(true) + .build() + .unwrap(), + ) { + Ok(_) => true, + Err(_) => false, + }; + + a && b && c } pub fn new( @@ -99,12 +111,17 @@ impl QueryRouter { let len = buf.get_i32(); let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]); // Don't read the ternminating null - let rgx = match SHARDING_REGEX_RE.get() { + let sharding_key_rgx = match SHARDING_REGEX_RE.get() { Some(r) => r, None => return false, }; - if rgx.is_match(&query) { + let set_shard_rgx = match SET_SHARD_REGEX_RE.get() { + Some(r) => r, + None => return false, + }; + + if sharding_key_rgx.is_match(&query) { let shard = query.split("'").collect::>()[1]; match shard.parse::() { @@ -120,6 +137,15 @@ impl QueryRouter { // case anyway. Err(_) => false, } + } else if set_shard_rgx.is_match(&query) { + let shard = query.split("'").collect::>()[1]; + match shard.parse::() { + Ok(shard) => { + self.active_shard = Some(shard); + true + } + Err(_) => false, + } } else { false } @@ -439,4 +465,28 @@ mod test { assert!(query_router.infer_role(res)); assert_eq!(query_router.role(), Some(Role::Replica)); } + + #[test] + fn test_set_shard_explicitely() { + QueryRouter::setup(); + + let default_server_role: Option = None; + let shards = 5; + + let mut query_router = QueryRouter::new(default_server_role, shards, false, false); + + // Build the special syntax query. + let mut message = BytesMut::new(); + let query = BytesMut::from(&b"SET SHARD TO '1'\0"[..]); + + message.put_u8(b'Q'); // Query + message.put_i32(query.len() as i32 + 4); + message.put_slice(&query[..]); + + assert!(query_router.select_shard(message)); + assert_eq!(query_router.shard(), 1); // See sharding.rs (we are using 5 shards on purpose in this test) + + query_router.reset(); + assert_eq!(query_router.shard(), 0); + } } diff --git a/tests/ruby/.ruby-version b/tests/ruby/.ruby-version new file mode 100644 index 000000000..860487ca1 --- /dev/null +++ b/tests/ruby/.ruby-version @@ -0,0 +1 @@ +2.7.1 diff --git a/tests/ruby/Gemfile b/tests/ruby/Gemfile new file mode 100644 index 000000000..94fb8c3d3 --- /dev/null +++ b/tests/ruby/Gemfile @@ -0,0 +1,4 @@ +source "https://rubygems.org" + +gem "pg" +gem "activerecord" diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock new file mode 100644 index 000000000..7b7d44841 --- /dev/null +++ b/tests/ruby/Gemfile.lock @@ -0,0 +1,30 @@ +GEM + remote: https://rubygems.org/ + specs: + activemodel (7.0.2.2) + activesupport (= 7.0.2.2) + activerecord (7.0.2.2) + activemodel (= 7.0.2.2) + activesupport (= 7.0.2.2) + activesupport (7.0.2.2) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + concurrent-ruby (1.1.9) + i18n (1.10.0) + concurrent-ruby (~> 1.0) + minitest (5.15.0) + pg (1.3.2) + tzinfo (2.0.4) + concurrent-ruby (~> 1.0) + +PLATFORMS + x86_64-linux + +DEPENDENCIES + activerecord + pg + +BUNDLED WITH + 2.3.7 diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index f12440fe4..18d1bfbf6 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -1,11 +1,74 @@ -require 'pg' +require "active_record" -conn = PG.connect(host: '127.0.0.1', port: 5433, dbname: 'test') +ActiveRecord.verbose_query_logs = true +ActiveRecord::Base.logger = Logger.new(STDOUT) -conn.exec( "SELECT * FROM pg_stat_activity" ) do |result| - puts " PID | User | Query" - result.each do |row| - puts " %7d | %-16s | %s " % - row.values_at('pid', 'usename', 'query') +ActiveRecord::Base.establish_connection( + adapter: "postgresql", + host: "127.0.0.1", + port: 6432, + username: "sharding_user", + password: "sharding_user", + database: "rails_dev", + prepared_statements: false, # Transaction mode + advisory_locks: false, # Same +) + +class TestTable < ActiveRecord::Base + self.table_name = "test_table" +end + +# # Create the table. +class CreateTestTable < ActiveRecord::Migration[7.0] + # Disable transasctions or things will fly out of order! + disable_ddl_transaction! + + SHARDS = 3 + + def change + SHARDS.times do |x| + # This will make this migration reversible! + reversible do + connection.execute "SET SHARD TO '#{x.to_i}'" + end + + # Always wrap the entire migration inside a transaction. If that's not possible, + # execute a `SET SHARD` command before every statement and make sure AR doesn't need + # to load database information beforehand (i.e. it's not the first query in the migration). + connection.transaction do + create_table :test_table, if_not_exists: true do |t| + t.string :name + t.string :description + + t.timestamps + end + end + end end -end \ No newline at end of file +end + +begin + CreateTestTable.migrate(:down) +rescue Exception + puts "Tables don't exist yet" +end + +CreateTestTable.migrate(:up) + +10.times do |x| + x += 1 # Postgres ids start at 1 + r = TestTable.connection.execute "SET SHARDING KEY TO '#{x.to_i}'" + + # Always wrap writes inside explicit transactions like these because ActiveRecord may fetch table info + # before actually issuing the `INSERT` statement. This ensures that that happens inside a transaction + # and the write goes to the correct shard. + TestTable.connection.transaction do + TestTable.create(id: x, name: "something_special_#{x.to_i}", description: "It's a surprise!") + end +end + +10.times do |x| + x += 1 # 0 confuses our sharding function + TestTable.connection.execute "SET SHARDING KEY TO '#{x.to_i}'" + puts TestTable.find_by_id(x).id +end From 885846b9364d8baba1f32cb361f9c5911cfea3ae Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 19 Feb 2022 08:57:24 -0800 Subject: [PATCH 091/551] More query router commands; settings last until changed again; docs (#25) * readme * touch up docs * stuff * refactor query router * remove unused * less verbose * docs * no link * method rename --- README.md | 166 +++++++++++----- src/client.rs | 67 ++++--- src/messages.rs | 93 +++++++++ src/query_router.rs | 474 ++++++++++++++++++++++---------------------- tests/ruby/tests.rb | 101 ++++++++-- 5 files changed, 567 insertions(+), 334 deletions(-) diff --git a/README.md b/README.md index 3f1c798ed..ab309c956 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,26 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su **Alpha**: don't use in production just yet. +## Features + +| **Feature** | **Status** | **Comments** | +|--------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| Transaction pooling | :heavy_check_mark: | Identical to PgBouncer. | +| Session pooling | :heavy_check_mark: | Identical to PgBouncer. | +| `COPY` support | :heavy_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | +| Query cancellation | :heavy_check_mark: | Supported both in transaction and session pooling modes. | +| Load balancing of read queries | :heavy_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | +| Sharding | :heavy_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | +| Failover | :heavy_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | +| Statistics reporting | :heavy_check_mark: | Statistics similar to PgBouncers are reported via StatsD. | +| Live configuration reloading | :x: :wrench: | On the roadmap; currently config changes require restart. | +| Client authentication | :x: :wrench: | On the roadmap; currently all clients are allowed to connect and one user is used to connect to Postgres. | + +## Deployment + +See `Dockerfile` for example deployment using Docker. The pooler is configured to spawn 4 workers so 4 CPUs are recommended for optimal performance. +That setting can be adjusted to spawn as many (or as little) workers as needed. + ## Local development 1. Install Rust (latest stable will work great). @@ -18,7 +38,7 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su ### Tests -You can just PgBench to test your changes: +Quickest way to test your changes is to use pgbench: ``` pgbench -i -h 127.0.0.1 -p 6432 && \ @@ -28,80 +48,130 @@ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended See [sharding README](./tests/sharding/README.md) for sharding logic testing. -## Features +| **Feature** | **Tested in CI** | **Tested manually** | **Comments** | +|----------------------|--------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------| +| Transaction pooling | :heavy_check_mark: | :heavy_check_mark: | Used by default for all tests. | +| Session pooling | :x: | :heavy_check_mark: | Easiest way to test is to enable it and run pgbench - results will be better than transaction pooling as expected. | +| `COPY` | :heavy_check_mark: | :heavy_check_mark: | `pgbench -i` uses `COPY`. `COPY FROM` is tested as well. | +| Query cancellation | :heavy_check_mark: | :heavy_check_mark: | `psql -c 'SELECT pg_sleep(1000);'` and press `Ctrl-C`. | +| Load balancing | :x: | :heavy_check_mark: | We could test this by emitting statistics for each replica and compare them. | +| Failover | :x: | :heavy_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing could include using Toxiproxy. | +| Sharding | :heavy_check_mark: | :heavy_check_mark: | See `tests/sharding` and `tests/ruby` for an Rails/ActiveRecord example. | +| Statistics reporting | :x: | :heavy_check_mark: | Run `nc -l -u 8125` and watch the stats come in every 15 seconds. | + -1. Session mode. -2. Transaction mode. -3. `COPY` protocol support. -4. Query cancellation. -5. Round-robin load balancing of replicas. -6. Banlist & failover. -7. Sharding! -8. Explicit query routing to primary or replicas. +## Usage ### Session mode -Each client owns its own server for the duration of the session. Commands like `SET` are allowed. -This is identical to PgBouncer session mode. +In session mode, a client talks to one server for the duration of the connection. Prepared statements, `SET`, and advisory locks are supported. In terms of supported features, there is very little if any difference between session mode and talking directly to the server. + +To use session mode, change `pool_mode = "session"`. ### Transaction mode -The connection is attached to the server for the duration of the transaction. `SET` will pollute the connection, -but `SET LOCAL` works great. Identical to PgBouncer transaction mode. +In transaction mode, a client talks to one server for the duration of a single transaction; once it's over, the server is returned to the pool. Prepared statements, `SET`, and advisory locks are not supported; alternatives are to use `SET LOCAL` and `pg_advisory_xact_lock` which are scoped to the transaction. -### COPY protocol -That one isn't particularly special, but good to mention that you can `COPY` data in and from the server -using this pooler. +This mode is enabled by default. -### Query cancellation -Okay, this is just basic stuff, but we support cancelling queries. If you know the Postgres protocol, -this might be relevant given than this is a transactional pooler but if you're new to Pg, don't worry about it, it works. +### Load balancing of read queries +All queries are load balanced against the configured servers using the round-robin algorithm. The most straight forward configuration example would be to put this pooler in front of several replicas and let it load balance all queries. -### Round-robin load balancing -This is the novel part. PgBouncer doesn't support it and suggests we use DNS or a TCP proxy instead. -We prefer to have everything as part of one package; arguably, it's easier to understand and optimize. -This pooler will round-robin between multiple replicas keeping load reasonably even. If the primary is in -the pool as well, it'll be treated as a replica for read-only queries. +If the configuration includes a primary and replicas, the queries can be separated with the built-in query parser. The query parser will interpret the query and route all `SELECT` queries to a replica, while all other queries including explicit transactions will be routed to the primary. -### Banlist & failover -This is where it gets even more interesting. If we fail to connect to one of the replicas or it fails a health check, -we add it to a ban list. No more new transactions will be served by that replica for, in our case, 60 seconds. This -gives it the opportunity to recover while clients are happily served by the remaining replicas. +The query parser is disabled by default. -This decreases error rates substantially! Worth noting here that on busy systems, if the replicas are running too hot, -failing over could bring even more load and tip over the remaining healthy-ish replicas. In this case, a decision should be made: -either lose 1/x of your traffic or risk losing it all eventually. Ideally you overprovision your system, so you don't necessarily need -to make this choice :-). +#### Query parser +The query parser will do its best to determine where the query should go, but sometimes that's not possible. In that case, the client can select which server it wants using this custom SQL syntax: -### Sharding -We're implemeting Postgres' `PARTITION BY HASH` sharding function for `BIGINT` fields. This works well for tables that use `BIGSERIAL` primary key which I think is common enough these days. We can also add many more functions here, but this is a good start. See `src/sharding.rs` and `tests/sharding/partition_hash_test_setup.sql` for more details on the implementation. +```sql +-- To talk to the primary for the duration of the next transaction: +SET SERVER ROLE TO 'primary'; + +-- To talk to the replica for the duration of the next transaction: +SET SERVER ROLE TO 'replica'; + +-- Let the query parser decide +SET SERVER ROLE TO 'auto'; + +-- Pick any server at random +SET SERVER ROLE TO 'any'; + +-- Reset to default configured settings +SET SERVER ROLE TO 'default'; +``` -The biggest advantage of using this sharding function is that anyone can shard the dataset using Postgres partitions -while also access it for both reads and writes using this pooler. No custom obscure sharding function is needed and database sharding can be done entirely in Postgres. +The setting will persist until it's changed again or the client disconnects. -To select the shard we want to talk to, we introduced special syntax: +By default, all queries are routed to all servers; `default_role` setting controls this behavior. + +### Failover +All servers are checked with a `SELECT 1` query before being given to a client. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the ban list is cleared: this is a safety precaution against false positives. The primary can never be banned. + +The ban time can be changed with `ban_time`. The default is 60 seconds. + +### Sharding +We use the `PARTITION BY HASH` hashing function, the same as used by Postgres for declarative partitioning. This allows to shard the database using Postgres partitions and place the partitions on different servers (shards). Both read and write queries can be routed to the shards using this pooler. + +To route queries to a particular shard, we use this custom SQL syntax: ```sql +-- To talk to a shard explicitely +SET SHARD TO '1'; + +-- To let the pooler choose based on a value SET SHARDING KEY TO '1234'; ``` -This sharding key will be hashed and the pooler will select a shard to use for the next transaction. If the pooler is in session mode, this sharding key has to be set as the first query on startup & cannot be changed until the client re-connects. +The active shard will last until it's changed again or the client disconnects. By default, the queries are routed to shard 0. -### Explicit read/write query routing +For hash function implementation, see `src/sharding.rs` and `tests/sharding/partition_hash_test_setup.sql`. -If you want to have the primary and replicas in the same pooler, you'd probably want to -route queries explicitely to the primary or replicas, depending if they are reads or writes (e.g `SELECT`s or `INSERT`/`UPDATE`, etc). To help with this, we introduce some more custom syntax: +#### ActiveRecord/Rails -```sql -SET SERVER ROLE TO 'primary'; -SET SERVER ROLE TO 'replica'; +```ruby +class User < ActiveRecord::Base +end + +# Metadata will be fetched from shard 0 +ActiveRecord::Base.establish_connection + +# Grab a bunch of users from shard 1 +User.connection.execute "SET SHARD TO '1'" +User.take(10) + +# Using id as the sharding key +User.connection.execute "SET SHARDING KEY TO '1234'" +User.find_by_id(1234) + +# Using geographical sharding +User.connection.execute "SET SERVER ROLE TO 'primary'" +User.connection.execute "SET SHARDING KEY TO '85'" +User.create(name: "test user", email: "test@example.com", zone_id: 85) + +# Let the query parser figure out where the query should go. +# We are still on shard = hash(85) % shards. +User.connection.execute "SET SERVER ROLE TO 'auto'" +User.find_by_email("test@example.com") ``` -After executing this, the next transaction will be routed to the primary or replica respectively. By default, all queries will be load-balanced between all servers, so if the client wants to write or talk to the primary, they have to explicitely select it using the syntax above. +#### Raw SQL +```sql +-- Grab a bunch of users from shard 1 +SET SHARD TO '1'; +SELECT * FROM users LIMT 10; +-- Find by id +SET SHARDING KEY TO '1234'; +SELECT * FROM USERS WHERE id = 1234; -## Missing +-- Writing in a primary/replicas configuration. +SET SHARDING ROLE TO 'primary'; +SET SHARDING KEY TO '85'; +INSERT INTO users (name, email, zome_id) VALUES ('test user', 'test@example.com', 85); -1. Authentication, ehem, this proxy is letting anyone in at the moment. +SET SERVER ROLE TO 'auto'; -- let the query router figure out where the query should go +SELECT * FROM users WHERE email = 'test@example.com'; -- shard setting lasts until set again; we are reading from the primary +``` ## Benchmarks diff --git a/src/client.rs b/src/client.rs index 184d46d9f..834bde0c3 100644 --- a/src/client.rs +++ b/src/client.rs @@ -14,7 +14,7 @@ use crate::constants::*; use crate::errors::Error; use crate::messages::*; use crate::pool::{ClientServerMap, ConnectionPool}; -use crate::query_router::QueryRouter; +use crate::query_router::{Command, QueryRouter}; use crate::server::Server; use crate::stats::Reporter; @@ -198,29 +198,50 @@ impl Client { // SET SHARDING KEY TO 'bigint'; let mut message = read_message(&mut self.read).await?; - // Parse for special select shard command. - // SET SHARDING KEY TO 'bigint'; - if query_router.select_shard(message.clone()) { - custom_protocol_response_ok( + // Handle all custom protocol commands here. + match query_router.try_execute_command(message.clone()) { + // Normal query + None => { + if query_router.query_parser_enabled() && query_router.role() == None { + query_router.infer_role(message.clone()); + } + } + + Some((Command::SetShard, _)) | Some((Command::SetShardingKey, _)) => { + custom_protocol_response_ok(&mut self.write, &format!("SET SHARD")).await?; + continue; + } + + Some((Command::SetServerRole, _)) => { + custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; + continue; + } + + Some((Command::ShowServerRole, value)) => { + show_response(&mut self.write, "server role", &value).await?; + continue; + } + + Some((Command::ShowShard, value)) => { + show_response(&mut self.write, "shard", &value).await?; + continue; + } + }; + + // Make sure we selected a valid shard. + if query_router.shard() >= pool.shards() { + error_response( &mut self.write, - &format!("SET SHARD TO {}", query_router.shard()), + &format!( + "shard '{}' is more than configured '{}'", + query_router.shard(), + pool.shards() + ), ) .await?; continue; } - // Parse for special server role selection command. - // SET SERVER ROLE TO '(primary|replica)'; - if query_router.select_role(message.clone()) { - custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; - continue; - } - - // Attempt to parse the query to determine where it should go - if query_router.query_parser_enabled() && query_router.role() == None { - query_router.infer_role(message.clone()); - } - // Grab a server from the pool: the client issued a regular query. let connection = match pool.get(query_router.shard(), query_router.role()).await { Ok(conn) => conn, @@ -228,7 +249,6 @@ impl Client { println!(">> Could not get connection from pool: {:?}", err); error_response(&mut self.write, "could not get connection from the pool") .await?; - query_router.reset(); continue; } }; @@ -310,9 +330,6 @@ impl Client { if self.transaction_mode { // Report this client as idle. self.stats.client_idle(); - - query_router.reset(); - break; } } @@ -395,9 +412,6 @@ impl Client { if self.transaction_mode { self.stats.client_idle(); - - query_router.reset(); - break; } } @@ -431,8 +445,7 @@ impl Client { self.stats.transaction(); if self.transaction_mode { - query_router.reset(); - + self.stats.client_idle(); break; } } diff --git a/src/messages.rs b/src/messages.rs index 3b5914d06..70dbc10e3 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -38,6 +38,16 @@ pub async fn backend_key_data( Ok(write_all(stream, key_data).await?) } +pub fn simple_query(query: &str) -> BytesMut { + let mut res = BytesMut::from(&b"Q"[..]); + let query = format!("{}\0", query); + + res.put_i32(query.len() as i32 + 4); + res.put_slice(&query.as_bytes()); + + res +} + /// Tell the client we're ready for another query. pub async fn ready_for_query(stream: &mut TcpStream) -> Result<(), Error> { let mut bytes = BytesMut::with_capacity(5); @@ -229,6 +239,89 @@ pub async fn error_response(stream: &mut OwnedWriteHalf, message: &str) -> Resul Ok(write_all_half(stream, res).await?) } +/// Respond to a SHOW SHARD command. +pub async fn show_response( + stream: &mut OwnedWriteHalf, + name: &str, + value: &str, +) -> Result<(), Error> { + // A SELECT response consists of: + // 1. RowDescription + // 2. One or more DataRow + // 3. CommandComplete + // 4. ReadyForQuery + + // RowDescription + let mut row_desc = BytesMut::new(); + + // Number of columns: 1 + row_desc.put_i16(1); + + // Column name + row_desc.put_slice(&format!("{}\0", name).as_bytes()); + + // Doesn't belong to any table + row_desc.put_i32(0); + + // Doesn't belong to any table + row_desc.put_i16(0); + + // Text + row_desc.put_i32(25); + + // Text size = variable (-1) + row_desc.put_i16(-1); + + // Type modifier: none that I know + row_desc.put_i32(0); + + // Format being used: text (0), binary (1) + row_desc.put_i16(0); + + // DataRow + let mut data_row = BytesMut::new(); + + // Number of columns + data_row.put_i16(1); + + // Size of the column content (length of the string really) + data_row.put_i32(value.len() as i32); + + // The content + data_row.put_slice(value.as_bytes()); + + // CommandComplete + let mut command_complete = BytesMut::new(); + + // Number of rows returned (just one) + command_complete.put_slice(&b"SELECT 1\0"[..]); + + // The final messages sent to the client + let mut res = BytesMut::new(); + + // RowDescription + res.put_u8(b'T'); + res.put_i32(row_desc.len() as i32 + 4); + res.put(row_desc); + + // DataRow + res.put_u8(b'D'); + res.put_i32(data_row.len() as i32 + 4); + res.put(data_row); + + // CommandComplete + res.put_u8(b'C'); + res.put_i32(command_complete.len() as i32 + 4); + res.put(command_complete); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} + /// Write all data in the buffer to the TcpStream. pub async fn write_all(stream: &mut TcpStream, buf: BytesMut) -> Result<(), Error> { match stream.write_all(&buf).await { diff --git a/src/query_router.rs b/src/query_router.rs index c02e6a615..c43ca4973 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1,22 +1,32 @@ +use crate::config::Role; +use crate::sharding::Sharder; /// Route queries automatically based on explicitely requested /// or implied query characteristics. use bytes::{Buf, BytesMut}; use once_cell::sync::OnceCell; -use regex::{Regex, RegexBuilder}; +use regex::RegexSet; use sqlparser::ast::Statement::{Query, StartTransaction}; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; -use crate::config::Role; -use crate::sharding::Sharder; - -const SHARDING_REGEX: &str = r"SET SHARDING KEY TO '[0-9]+'"; -const SET_SHARD_REGEX: &str = r"SET SHARD TO '[0-9]+'"; -const ROLE_REGEX: &str = r"SET SERVER ROLE TO '(PRIMARY|REPLICA)'"; +const CUSTOM_SQL_REGEXES: [&str; 5] = [ + r"(?i)SET SHARDING KEY TO '[0-9]+'", + r"(?i)SET SHARD TO '[0-9]+'", + r"(?i)SHOW SHARD", + r"(?i)SET SERVER ROLE TO '(PRIMARY|REPLICA|ANY|AUTO|DEFAULT)'", + r"(?i)SHOW SERVER ROLE", +]; + +#[derive(PartialEq, Debug)] +pub enum Command { + SetShardingKey, + SetShard, + ShowShard, + SetServerRole, + ShowServerRole, +} -static SHARDING_REGEX_RE: OnceCell = OnceCell::new(); -static ROLE_REGEX_RE: OnceCell = OnceCell::new(); -static SET_SHARD_REGEX_RE: OnceCell = OnceCell::new(); +static CUSTOM_SQL_REGEX_SET: OnceCell = OnceCell::new(); pub struct QueryRouter { // By default, queries go here, unless we have better information @@ -41,38 +51,18 @@ pub struct QueryRouter { impl QueryRouter { pub fn setup() -> bool { - // Compile our query routing regexes early, so we only do it once. - let a = match SHARDING_REGEX_RE.set( - RegexBuilder::new(SHARDING_REGEX) - .case_insensitive(true) - .build() - .unwrap(), - ) { - Ok(_) => true, - Err(_) => false, - }; - - let b = match ROLE_REGEX_RE.set( - RegexBuilder::new(ROLE_REGEX) - .case_insensitive(true) - .build() - .unwrap(), - ) { - Ok(_) => true, - Err(_) => false, + let set = match RegexSet::new(&CUSTOM_SQL_REGEXES) { + Ok(rgx) => rgx, + Err(err) => { + log::error!("QueryRouter::setup Could not compile regex set: {:?}", err); + return false; + } }; - let c = match SET_SHARD_REGEX_RE.set( - RegexBuilder::new(SET_SHARD_REGEX) - .case_insensitive(true) - .build() - .unwrap(), - ) { + match CUSTOM_SQL_REGEX_SET.set(set) { Ok(_) => true, Err(_) => false, - }; - - a && b && c + } } pub fn new( @@ -92,104 +82,104 @@ impl QueryRouter { } } - /// Determine if the query is part of our special syntax, extract - /// the shard key, and return the shard to query based on Postgres' - /// PARTITION BY HASH function. - pub fn select_shard(&mut self, mut buf: BytesMut) -> bool { + /// Try to parse a command and execute it. + pub fn try_execute_command(&mut self, mut buf: BytesMut) -> Option<(Command, String)> { let code = buf.get_u8() as char; - // Only supporting simpe protocol here, so - // one would have to execute something like this: - // psql -c "SET SHARDING KEY TO '1234'" - // after sanitizing the value manually, which can be just done with an - // int parser, e.g. `let key = "1234".parse::().unwrap()`. - match code { - 'Q' => (), - _ => return false, - }; - - let len = buf.get_i32(); - let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]); // Don't read the ternminating null + if code != 'Q' { + return None; + } - let sharding_key_rgx = match SHARDING_REGEX_RE.get() { - Some(r) => r, - None => return false, - }; + let len = buf.get_i32() as usize; + let query = String::from_utf8_lossy(&buf[..len - 5]).to_string(); // Ignore the terminating NULL. - let set_shard_rgx = match SET_SHARD_REGEX_RE.get() { - Some(r) => r, - None => return false, + let regex_set = match CUSTOM_SQL_REGEX_SET.get() { + Some(regex_set) => regex_set, + None => return None, }; - if sharding_key_rgx.is_match(&query) { - let shard = query.split("'").collect::>()[1]; - - match shard.parse::() { - Ok(shard) => { - let sharder = Sharder::new(self.shards); - self.active_shard = Some(sharder.pg_bigint_hash(shard)); + let matches: Vec<_> = regex_set.matches(&query).into_iter().collect(); - true - } - - // The shard must be a valid integer. Our regex won't let anything else pass, - // so this code will never run, but Rust can't know that, so we have to handle this - // case anyway. - Err(_) => false, - } - } else if set_shard_rgx.is_match(&query) { - let shard = query.split("'").collect::>()[1]; - match shard.parse::() { - Ok(shard) => { - self.active_shard = Some(shard); - true - } - Err(_) => false, - } - } else { - false + if matches.len() != 1 { + return None; } - } - /// Pick a primary or a replica from the pool. - pub fn select_role(&mut self, mut buf: BytesMut) -> bool { - let code = buf.get_u8() as char; - - // Same story as select_shard() above. - match code { - 'Q' => (), - _ => return false, + let command = match matches[0] { + 0 => Command::SetShardingKey, + 1 => Command::SetShard, + 2 => Command::ShowShard, + 3 => Command::SetServerRole, + 4 => Command::ShowServerRole, + _ => unreachable!(), }; - let len = buf.get_i32(); - let query = String::from_utf8_lossy(&buf[..len as usize - 4 - 1]).to_ascii_uppercase(); + let mut value = match command { + Command::SetShardingKey | Command::SetShard | Command::SetServerRole => { + query.split("'").collect::>()[1].to_string() + } - let rgx = match ROLE_REGEX_RE.get() { - Some(r) => r, - None => return false, + Command::ShowShard => self.shard().to_string(), + Command::ShowServerRole => match self.active_role { + Some(Role::Primary) => String::from("primary"), + Some(Role::Replica) => String::from("replica"), + None => { + if self.query_parser_enabled { + String::from("auto") + } else { + String::from("any") + } + } + }, }; - // Copy / paste from above. If we get one more of these use cases, - // it'll be time to abstract :). - if rgx.is_match(&query) { - let role = query.split("'").collect::>()[1]; + match command { + Command::SetShardingKey => { + let sharder = Sharder::new(self.shards); + let shard = sharder.pg_bigint_hash(value.parse::().unwrap()); + self.active_shard = Some(shard); + value = shard.to_string(); + } - match role { - "PRIMARY" => { - self.active_role = Some(Role::Primary); - true - } - "REPLICA" => { - self.active_role = Some(Role::Replica); - true - } + Command::SetShard => { + self.active_shard = Some(value.parse::().unwrap()); + } - // Our regex won't let this case happen, but Rust can't know that. - _ => false, + Command::SetServerRole => { + self.active_role = match value.to_ascii_lowercase().as_ref() { + "primary" => { + self.query_parser_enabled = false; + Some(Role::Primary) + } + + "replica" => { + self.query_parser_enabled = false; + Some(Role::Replica) + } + + "any" => { + self.query_parser_enabled = false; + None + } + + "auto" => { + self.query_parser_enabled = true; + None + } + + "default" => { + // TODO: reset query parser to default here. + self.active_role = self.default_server_role; + self.active_role + } + + _ => unreachable!(), + }; } - } else { - false + + _ => (), } + + Some((command, value)) } /// Try to infer which server to connect to based on the contents of the query. @@ -270,13 +260,13 @@ impl QueryRouter { pub fn shard(&self) -> usize { match self.active_shard { Some(shard) => shard, - None => 0, // TODO: pick random shard + None => 0, } } /// Reset the router back to defaults. /// This must be called at the end of every transaction in transaction mode. - pub fn reset(&mut self) { + pub fn _reset(&mut self) { self.active_role = self.default_server_role; self.active_shard = None; } @@ -290,87 +280,18 @@ impl QueryRouter { #[cfg(test)] mod test { use super::*; + use crate::messages::simple_query; use bytes::BufMut; - #[test] - fn test_select_shard() { - QueryRouter::setup(); - - let default_server_role: Option = None; - let shards = 5; - let mut query_router = QueryRouter::new(default_server_role, shards, false, false); - - // Build the special syntax query. - let mut message = BytesMut::new(); - let query = BytesMut::from(&b"SET SHARDING KEY TO '13';\0"[..]); - - message.put_u8(b'Q'); // Query - message.put_i32(query.len() as i32 + 4); - message.put_slice(&query[..]); - - assert!(query_router.select_shard(message)); - assert_eq!(query_router.shard(), 3); // See sharding.rs (we are using 5 shards on purpose in this test) - - query_router.reset(); - assert_eq!(query_router.shard(), 0); - } - - #[test] - fn test_select_replica() { - QueryRouter::setup(); - - let default_server_role: Option = None; - let shards = 5; - let mut query_router = QueryRouter::new(default_server_role, shards, false, false); - - // Build the special syntax query. - let mut message = BytesMut::new(); - let query = BytesMut::from(&b"SET SERVER ROLE TO 'replica';\0"[..]); - - message.put_u8(b'Q'); // Query - message.put_i32(query.len() as i32 + 4); - message.put_slice(&query[..]); - - assert!(query_router.select_role(message)); - assert_eq!(query_router.role(), Some(Role::Replica)); - - query_router.reset(); - - assert_eq!(query_router.role(), default_server_role); - } - #[test] fn test_defaults() { QueryRouter::setup(); let default_server_role: Option = None; let shards = 5; - let query_router = QueryRouter::new(default_server_role, shards, false, false); - - assert_eq!(query_router.shard(), 0); - assert_eq!(query_router.role(), None); - } - - #[test] - fn test_incorrect_syntax() { - QueryRouter::setup(); - - let default_server_role: Option = None; - let shards = 5; - let mut query_router = QueryRouter::new(default_server_role, shards, false, false); + let qr = QueryRouter::new(default_server_role, shards, false, false); - // Build the special syntax query. - let mut message = BytesMut::new(); - - // Typo! - let query = BytesMut::from(&b"SET SERVER RLE TO 'replica';\0"[..]); - - message.put_u8(b'Q'); // Query - message.put_i32(query.len() as i32 + 4); - message.put_slice(&query[..]); - - assert_eq!(query_router.select_shard(message.clone()), false); - assert_eq!(query_router.select_role(message.clone()), false); + assert_eq!(qr.role(), None); } #[test] @@ -379,23 +300,20 @@ mod test { let default_server_role: Option = None; let shards = 5; - - let mut query_router = QueryRouter::new(default_server_role, shards, false, false); + let mut qr = QueryRouter::new(default_server_role, shards, false, false); let queries = vec![ - BytesMut::from(&b"SELECT * FROM items WHERE id = 5\0"[..]), - BytesMut::from(&b"SELECT id, name, value FROM items INNER JOIN prices ON item.id = prices.item_id\0"[..]), - BytesMut::from(&b"WITH t AS (SELECT * FROM items) SELECT * FROM t\0"[..]), + simple_query("SELECT * FROM items WHERE id = 5"), + simple_query( + "SELECT id, name, value FROM items INNER JOIN prices ON item.id = prices.item_id", + ), + simple_query("WITH t AS (SELECT * FROM items) SELECT * FROM t"), ]; - for query in &queries { - let mut res = BytesMut::from(&b"Q"[..]); - res.put_i32(query.len() as i32 + 4); - res.put(query.clone()); - + for query in queries { // It's a recognized query - assert!(query_router.infer_role(res)); - assert_eq!(query_router.role(), Some(Role::Replica)); + assert!(qr.infer_role(query)); + assert_eq!(qr.role(), Some(Role::Replica)); } } @@ -405,24 +323,19 @@ mod test { let default_server_role: Option = None; let shards = 5; - - let mut query_router = QueryRouter::new(default_server_role, shards, false, false); + let mut qr = QueryRouter::new(default_server_role, shards, false, false); let queries = vec![ - BytesMut::from(&b"UPDATE items SET name = 'pumpkin' WHERE id = 5\0"[..]), - BytesMut::from(&b"INSERT INTO items (id, name) VALUES (5, 'pumpkin')\0"[..]), - BytesMut::from(&b"DELETE FROM items WHERE id = 5\0"[..]), - BytesMut::from(&b"BEGIN\0"[..]), // Transaction start + simple_query("UPDATE items SET name = 'pumpkin' WHERE id = 5"), + simple_query("INSERT INTO items (id, name) VALUES (5, 'pumpkin')"), + simple_query("DELETE FROM items WHERE id = 5"), + simple_query("BEGIN"), // Transaction start ]; - for query in &queries { - let mut res = BytesMut::from(&b"Q"[..]); - res.put_i32(query.len() as i32 + 4); - res.put(query.clone()); - + for query in queries { // It's a recognized query - assert!(query_router.infer_role(res)); - assert_eq!(query_router.role(), Some(Role::Primary)); + assert!(qr.infer_role(query)); + assert_eq!(qr.role(), Some(Role::Primary)); } } @@ -432,16 +345,11 @@ mod test { let default_server_role: Option = None; let shards = 5; + let mut qr = QueryRouter::new(default_server_role, shards, true, false); + let query = simple_query("SELECT * FROM items WHERE id = 5"); - let mut query_router = QueryRouter::new(default_server_role, shards, true, false); - - let query = BytesMut::from(&b"SELECT * FROM items WHERE id = 5\0"[..]); - let mut res = BytesMut::from(&b"Q"[..]); - res.put_i32(query.len() as i32 + 4); - res.put(query.clone()); - - assert!(query_router.infer_role(res)); - assert_eq!(query_router.role(), None); + assert!(qr.infer_role(query)); + assert_eq!(qr.role(), None); } #[test] @@ -467,26 +375,112 @@ mod test { } #[test] - fn test_set_shard_explicitely() { + fn test_regex_set() { QueryRouter::setup(); - let default_server_role: Option = None; - let shards = 5; + let tests = [ + // Upper case + "SET SHARDING KEY TO '1'", + "SET SHARD TO '1'", + "SHOW SHARD", + "SET SERVER ROLE TO 'replica'", + "SET SERVER ROLE TO 'primary'", + "SET SERVER ROLE TO 'any'", + "SET SERVER ROLE TO 'auto'", + "SHOW SERVER ROLE", + // Lower case + "set sharding key to '1'", + "set shard to '1'", + "show shard", + "set server role to 'replica'", + "set server role to 'primary'", + "set server role to 'any'", + "set server role to 'auto'", + "show server role", + ]; - let mut query_router = QueryRouter::new(default_server_role, shards, false, false); + let set = CUSTOM_SQL_REGEX_SET.get().unwrap(); + + for test in &tests { + let matches: Vec<_> = set.matches(test).into_iter().collect(); + + assert_eq!(matches.len(), 1); + } + } + + #[test] + fn test_try_execute_command() { + QueryRouter::setup(); + let mut qr = QueryRouter::new(Some(Role::Primary), 5, false, false); + + // SetShardingKey + let query = simple_query("SET SHARDING KEY TO '13'"); + assert_eq!( + qr.try_execute_command(query), + Some((Command::SetShardingKey, String::from("3"))) + ); + assert_eq!(qr.shard(), 3); + + // SetShard + let query = simple_query("SET SHARD TO '1'"); + assert_eq!( + qr.try_execute_command(query), + Some((Command::SetShard, String::from("1"))) + ); + assert_eq!(qr.shard(), 1); + + // ShowShard + let query = simple_query("SHOW SHARD"); + assert_eq!( + qr.try_execute_command(query), + Some((Command::ShowShard, String::from("1"))) + ); - // Build the special syntax query. - let mut message = BytesMut::new(); - let query = BytesMut::from(&b"SET SHARD TO '1'\0"[..]); + // SetServerRole + let roles = ["primary", "replica", "any", "auto", "primary"]; + let verify_roles = [ + Some(Role::Primary), + Some(Role::Replica), + None, + None, + Some(Role::Primary), + ]; + let query_parser_enabled = [false, false, false, true, false]; + + for (idx, role) in roles.iter().enumerate() { + let query = simple_query(&format!("SET SERVER ROLE TO '{}'", role)); + assert_eq!( + qr.try_execute_command(query), + Some((Command::SetServerRole, String::from(*role))) + ); + assert_eq!(qr.role(), verify_roles[idx],); + assert_eq!(qr.query_parser_enabled(), query_parser_enabled[idx],); + + // ShowServerRole + let query = simple_query("SHOW SERVER ROLE"); + assert_eq!( + qr.try_execute_command(query), + Some((Command::ShowServerRole, String::from(*role))) + ); + } + } + + #[test] + fn test_enable_query_parser() { + QueryRouter::setup(); + let mut qr = QueryRouter::new(None, 5, false, false); + let query = simple_query("SET SERVER ROLE TO 'auto'"); - message.put_u8(b'Q'); // Query - message.put_i32(query.len() as i32 + 4); - message.put_slice(&query[..]); + assert!(qr.try_execute_command(query) != None); + assert!(qr.query_parser_enabled()); + assert_eq!(qr.role(), None); - assert!(query_router.select_shard(message)); - assert_eq!(query_router.shard(), 1); // See sharding.rs (we are using 5 shards on purpose in this test) + let query = simple_query("INSERT INTO test_table VALUES (1)"); + assert_eq!(qr.infer_role(query), true); + assert_eq!(qr.role(), Some(Role::Primary)); - query_router.reset(); - assert_eq!(query_router.shard(), 0); + let query = simple_query("SELECT * FROM test_table"); + assert_eq!(qr.infer_role(query), true); + assert_eq!(qr.role(), Some(Role::Replica)); } } diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index 18d1bfbf6..3ff39ab16 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -1,7 +1,8 @@ require "active_record" -ActiveRecord.verbose_query_logs = true -ActiveRecord::Base.logger = Logger.new(STDOUT) +# Uncomment these two to see all queries. +# ActiveRecord.verbose_query_logs = true +# ActiveRecord::Base.logger = Logger.new(STDOUT) ActiveRecord::Base.establish_connection( adapter: "postgresql", @@ -18,6 +19,13 @@ class TestTable < ActiveRecord::Base self.table_name = "test_table" end +class TestSafeTable < ActiveRecord::Base + self.table_name = "test_safe_table" +end + +class ShouldNeverHappenException < Exception +end + # # Create the table. class CreateTestTable < ActiveRecord::Migration[7.0] # Disable transasctions or things will fly out of order! @@ -30,6 +38,7 @@ def change # This will make this migration reversible! reversible do connection.execute "SET SHARD TO '#{x.to_i}'" + connection.execute "SET SERVER ROLE TO 'primary'" end # Always wrap the entire migration inside a transaction. If that's not possible, @@ -47,28 +56,82 @@ def change end end -begin - CreateTestTable.migrate(:down) -rescue Exception - puts "Tables don't exist yet" +class CreateSafeShardedTable < ActiveRecord::Migration[7.0] + # Disable transasctions or things will fly out of order! + disable_ddl_transaction! + + SHARDS = 3 + + def up + SHARDS.times do |x| + # This will make this migration reversible! + connection.execute "SET SHARD TO '#{x.to_i}'" + connection.execute "SET SERVER ROLE TO 'primary'" + + connection.execute <<-SQL + CREATE TABLE test_safe_table ( + id BIGINT PRIMARY KEY, + name VARCHAR, + description TEXT + ) PARTITION BY HASH (id); + + CREATE TABLE test_safe_table_data PARTITION OF test_safe_table + FOR VALUES WITH (MODULUS #{SHARDS.to_i}, REMAINDER #{x.to_i}); + SQL + end + end + + def down + SHARDS.times do |x| + connection.execute "SET SHARD TO '#{x.to_i}'" + connection.execute "SET SERVER ROLE TO 'primary'" + connection.execute "DROP TABLE test_safe_table CASCADE" + end + end end -CreateTestTable.migrate(:up) +20.times do + begin + CreateTestTable.migrate(:down) + rescue Exception + puts "Tables don't exist yet" + end + + begin + CreateSafeShardedTable.migrate(:down) + rescue Exception + puts "Tables don't exist yet" + end + + CreateTestTable.migrate(:up) + CreateSafeShardedTable.migrate(:up) -10.times do |x| - x += 1 # Postgres ids start at 1 - r = TestTable.connection.execute "SET SHARDING KEY TO '#{x.to_i}'" + 3.times do |x| + TestSafeTable.connection.execute "SET SHARD TO '#{x.to_i}'" + TestSafeTable.connection.execute "SET SERVER ROLE TO 'primary'" + TestSafeTable.connection.execute "TRUNCATE #{TestTable.table_name}" + end - # Always wrap writes inside explicit transactions like these because ActiveRecord may fetch table info - # before actually issuing the `INSERT` statement. This ensures that that happens inside a transaction - # and the write goes to the correct shard. - TestTable.connection.transaction do - TestTable.create(id: x, name: "something_special_#{x.to_i}", description: "It's a surprise!") + 10.times do |x| + x += 1 # Postgres ids start at 1 + TestSafeTable.connection.execute "SET SHARDING KEY TO '#{x.to_i}'" + TestSafeTable.connection.execute "SET SERVER ROLE TO 'primary'" + TestSafeTable.create(id: x, name: "something_special_#{x.to_i}", description: "It's a surprise!") + end + + 10.times do |x| + x += 1 # 0 confuses our sharding function + TestSafeTable.connection.execute "SET SHARDING KEY TO '#{x.to_i}'" + TestSafeTable.connection.execute "SET SERVER ROLE TO 'replica'" + TestSafeTable.find_by_id(x).id end end -10.times do |x| - x += 1 # 0 confuses our sharding function - TestTable.connection.execute "SET SHARDING KEY TO '#{x.to_i}'" - puts TestTable.find_by_id(x).id +# Test wrong shard +TestSafeTable.connection.execute "SET SHARD TO '1'" +begin + TestSafeTable.create(id: 5, name: "test", description: "test description") + raise ShouldNeverHappenException("Uh oh") +rescue ActiveRecord::StatementInvalid + puts "OK" end From ccbee4997f08c778892472e871820058ff85376e Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 19 Feb 2022 08:59:20 -0800 Subject: [PATCH 092/551] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab309c956..1408d9b25 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ SET SERVER ROLE TO 'default'; The setting will persist until it's changed again or the client disconnects. -By default, all queries are routed to all servers; `default_role` setting controls this behavior. +By default, all queries are routed to the first available server; `default_role` setting controls this behavior. ### Failover All servers are checked with a `SELECT 1` query before being given to a client. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the ban list is cleared: this is a safety precaution against false positives. The primary can never be banned. From 02b5767213cfa6bf89774e161e0b20bc0a57b6ae Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 19 Feb 2022 13:57:35 -0800 Subject: [PATCH 093/551] Reloadable config (#26) * Reloadable config * readme * live config reload * test matrix --- .circleci/run_tests.sh | 3 ++ Cargo.lock | 7 +++ Cargo.toml | 1 + README.md | 71 +++++++++++++++++---------- pgcat.toml | 3 ++ src/client.rs | 13 ++--- src/config.rs | 106 +++++++++++++++++++++++++++++++++++++---- src/main.rs | 86 ++++++++++++++------------------- src/messages.rs | 1 + src/pool.rs | 14 +++--- src/query_router.rs | 74 ++++++++++++++-------------- src/stats.rs | 4 +- 12 files changed, 250 insertions(+), 133 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 8a7310281..9ba0686eb 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -34,6 +34,9 @@ psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql > / # Replica/primary selection & more sharding tests psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null +# Test reload config +kill -SIGHUP $(pgrep pgcat) + # # ActiveRecord tests! # diff --git a/Cargo.lock b/Cargo.lock index d11410a32..befc8f137 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "arc-swap" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" + [[package]] name = "async-trait" version = "0.1.52" @@ -318,6 +324,7 @@ dependencies = [ name = "pgcat" version = "0.1.0" dependencies = [ + "arc-swap", "async-trait", "bb8", "bytes", diff --git a/Cargo.toml b/Cargo.toml index ba78b59bd..7afcc6a8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,4 @@ once_cell = "1" statsd = "0.15" sqlparser = "0.14" log = "0.4" +arc-swap = "1" diff --git a/README.md b/README.md index 1408d9b25..cf955eb25 100644 --- a/README.md +++ b/README.md @@ -9,19 +9,18 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su **Alpha**: don't use in production just yet. ## Features - -| **Feature** | **Status** | **Comments** | -|--------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| -| Transaction pooling | :heavy_check_mark: | Identical to PgBouncer. | -| Session pooling | :heavy_check_mark: | Identical to PgBouncer. | -| `COPY` support | :heavy_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | -| Query cancellation | :heavy_check_mark: | Supported both in transaction and session pooling modes. | -| Load balancing of read queries | :heavy_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | -| Sharding | :heavy_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | -| Failover | :heavy_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | -| Statistics reporting | :heavy_check_mark: | Statistics similar to PgBouncers are reported via StatsD. | -| Live configuration reloading | :x: :wrench: | On the roadmap; currently config changes require restart. | -| Client authentication | :x: :wrench: | On the roadmap; currently all clients are allowed to connect and one user is used to connect to Postgres. | +| **Feature** | **Status** | **Comments** | +|--------------------------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| Transaction pooling | :heavy_check_mark: | Identical to PgBouncer. | +| Session pooling | :heavy_check_mark: | Identical to PgBouncer. | +| `COPY` support | :heavy_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | +| Query cancellation | :heavy_check_mark: | Supported both in transaction and session pooling modes. | +| Load balancing of read queries | :heavy_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | +| Sharding | :heavy_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | +| Failover | :heavy_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | +| Statistics reporting | :heavy_check_mark: | Statistics similar to PgBouncers are reported via StatsD. | +| Live configuration reloading | :construction_worker: | Reload config with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)`. Not all settings can be reloaded without a restart. | +| Client authentication | :x: :wrench: | On the roadmap; currently all clients are allowed to connect and one user is used to connect to Postgres. | ## Deployment @@ -48,17 +47,17 @@ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended See [sharding README](./tests/sharding/README.md) for sharding logic testing. -| **Feature** | **Tested in CI** | **Tested manually** | **Comments** | -|----------------------|--------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------| -| Transaction pooling | :heavy_check_mark: | :heavy_check_mark: | Used by default for all tests. | -| Session pooling | :x: | :heavy_check_mark: | Easiest way to test is to enable it and run pgbench - results will be better than transaction pooling as expected. | -| `COPY` | :heavy_check_mark: | :heavy_check_mark: | `pgbench -i` uses `COPY`. `COPY FROM` is tested as well. | -| Query cancellation | :heavy_check_mark: | :heavy_check_mark: | `psql -c 'SELECT pg_sleep(1000);'` and press `Ctrl-C`. | -| Load balancing | :x: | :heavy_check_mark: | We could test this by emitting statistics for each replica and compare them. | -| Failover | :x: | :heavy_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing could include using Toxiproxy. | -| Sharding | :heavy_check_mark: | :heavy_check_mark: | See `tests/sharding` and `tests/ruby` for an Rails/ActiveRecord example. | -| Statistics reporting | :x: | :heavy_check_mark: | Run `nc -l -u 8125` and watch the stats come in every 15 seconds. | - +| **Feature** | **Tested in CI** | **Tested manually** | **Comments** | +|-----------------------|--------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------| +| Transaction pooling | :heavy_check_mark: | :heavy_check_mark: | Used by default for all tests. | +| Session pooling | :x: | :heavy_check_mark: | Easiest way to test is to enable it and run pgbench - results will be better than transaction pooling as expected. | +| `COPY` | :heavy_check_mark: | :heavy_check_mark: | `pgbench -i` uses `COPY`. `COPY FROM` is tested as well. | +| Query cancellation | :heavy_check_mark: | :heavy_check_mark: | `psql -c 'SELECT pg_sleep(1000);'` and press `Ctrl-C`. | +| Load balancing | :x: | :heavy_check_mark: | We could test this by emitting statistics for each replica and compare them. | +| Failover | :x: | :heavy_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing could include using Toxiproxy. | +| Sharding | :heavy_check_mark: | :heavy_check_mark: | See `tests/sharding` and `tests/ruby` for an Rails/ActiveRecord example. | +| Statistics reporting | :x: | :heavy_check_mark: | Run `nc -l -u 8125` and watch the stats come in every 15 seconds. | +| Live config reloading | :heavy_check_mark: | :heavy_check_mark: | Run `kill -s SIGHUP $(pgrep pgcat)` and watch the config reload. | ## Usage @@ -173,6 +172,30 @@ SET SERVER ROLE TO 'auto'; -- let the query router figure out where the query sh SELECT * FROM users WHERE email = 'test@example.com'; -- shard setting lasts until set again; we are reading from the primary ``` +### Statistics reporting + +Stats are reported using StatsD every 15 seconds. The address is configurable with `statsd_address`, the default is `127.0.0.1:8125`. The stats are very similar to what Pgbouncer reports and the names are kept to be comparable. + +### Live configuration reloading + +The config can be reloaded by sending a `kill -s SIGHUP` to the process. Not all settings are currently supported by live reload: + +| **Config** | **Requires restart** | +|-------------------------|----------------------| +| `host` | yes | +| `port` | yes | +| `pool_mode` | no | +| `connect_timeout` | yes | +| `healthcheck_timeout` | no | +| `ban_time` | no | +| `statsd_address` | yes | +| `user` | yes | +| `shards` | yes | +| `default_role` | no | +| `primary_reads_enabled` | no | +| `query_parser_enabled` | no | + + ## Benchmarks You can setup PgBench locally through PgCat: diff --git a/pgcat.toml b/pgcat.toml index 0fa8b6a0a..9a240b3c9 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -29,6 +29,9 @@ healthcheck_timeout = 1000 # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # Seconds +# Stats will be sent here +statsd_address = "127.0.0.1:8125" + # # User to use for authentication against the server. [user] diff --git a/src/client.rs b/src/client.rs index 834bde0c3..1b921b84b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -10,6 +10,7 @@ use tokio::net::{ use std::collections::HashMap; +use crate::config::get_config; use crate::constants::*; use crate::errors::Error; use crate::messages::*; @@ -61,10 +62,12 @@ impl Client { pub async fn startup( mut stream: TcpStream, client_server_map: ClientServerMap, - transaction_mode: bool, server_info: BytesMut, stats: Reporter, ) -> Result { + let config = get_config(); + let transaction_mode = config.general.pool_mode.starts_with("t"); + drop(config); loop { // Could be StartupMessage or SSLRequest // which makes this variable length. @@ -154,11 +157,7 @@ impl Client { } /// Client loop. We handle all messages between the client and the database here. - pub async fn handle( - &mut self, - mut pool: ConnectionPool, - mut query_router: QueryRouter, - ) -> Result<(), Error> { + pub async fn handle(&mut self, mut pool: ConnectionPool) -> Result<(), Error> { // The client wants to cancel a query it has issued previously. if self.cancel_mode { let (process_id, secret_key, address, port) = { @@ -187,6 +186,8 @@ impl Client { return Ok(Server::cancel(&address, &port, process_id, secret_key).await?); } + let mut query_router = QueryRouter::new(); + // Our custom protocol loop. // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocols. diff --git a/src/config.rs b/src/config.rs index 8f7c45fa2..261d596f8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,12 +1,17 @@ +use arc_swap::{ArcSwap, Guard}; +use once_cell::sync::Lazy; use serde_derive::Deserialize; use tokio::fs::File; use tokio::io::AsyncReadExt; use toml; use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use crate::errors::Error; +static CONFIG: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Config::default())); + #[derive(Clone, PartialEq, Deserialize, Hash, std::cmp::Eq, Debug, Copy)] pub enum Role { Primary, @@ -39,12 +44,32 @@ pub struct Address { pub role: Role, } +impl Default for Address { + fn default() -> Address { + Address { + host: String::from("127.0.0.1"), + port: String::from("5432"), + shard: 0, + role: Role::Replica, + } + } +} + #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Deserialize, Debug)] pub struct User { pub name: String, pub password: String, } +impl Default for User { + fn default() -> User { + User { + name: String::from("postgres"), + password: String::new(), + } + } +} + #[derive(Deserialize, Debug, Clone)] pub struct General { pub host: String, @@ -54,6 +79,22 @@ pub struct General { pub connect_timeout: u64, pub healthcheck_timeout: u64, pub ban_time: i64, + pub statsd_address: String, +} + +impl Default for General { + fn default() -> General { + General { + host: String::from("localhost"), + port: 5432, + pool_size: 15, + pool_mode: String::from("transaction"), + connect_timeout: 5000, + healthcheck_timeout: 1000, + ban_time: 60, + statsd_address: String::from("127.0.0.1:8125"), + } + } } #[derive(Deserialize, Debug, Clone)] @@ -62,6 +103,15 @@ pub struct Shard { pub database: String, } +impl Default for Shard { + fn default() -> Shard { + Shard { + servers: vec![(String::from("localhost"), 5432, String::from("primary"))], + database: String::from("postgres"), + } + } +} + #[derive(Deserialize, Debug, Clone)] pub struct QueryRouter { pub default_role: String, @@ -69,6 +119,16 @@ pub struct QueryRouter { pub primary_reads_enabled: bool, } +impl Default for QueryRouter { + fn default() -> QueryRouter { + QueryRouter { + default_role: String::from("any"), + query_parser_enabled: false, + primary_reads_enabled: true, + } + } +} + #[derive(Deserialize, Debug, Clone)] pub struct Config { pub general: General, @@ -77,8 +137,36 @@ pub struct Config { pub query_router: QueryRouter, } +impl Default for Config { + fn default() -> Config { + Config { + general: General::default(), + user: User::default(), + shards: HashMap::from([(String::from("1"), Shard::default())]), + query_router: QueryRouter::default(), + } + } +} + +impl Config { + pub fn show(&self) { + println!("> Pool size: {}", self.general.pool_size); + println!("> Pool mode: {}", self.general.pool_mode); + println!("> Ban time: {}s", self.general.ban_time); + println!( + "> Healthcheck timeout: {}ms", + self.general.healthcheck_timeout + ); + println!("> Connection timeout: {}ms", self.general.connect_timeout); + } +} + +pub fn get_config() -> Guard> { + CONFIG.load() +} + /// Parse the config. -pub async fn parse(path: &str) -> Result { +pub async fn parse(path: &str) -> Result<(), Error> { let mut contents = String::new(); let mut file = match File::open(path).await { Ok(file) => file, @@ -163,7 +251,9 @@ pub async fn parse(path: &str) -> Result { } }; - Ok(config) + CONFIG.store(Arc::new(config.clone())); + + Ok(()) } #[cfg(test)] @@ -172,11 +262,11 @@ mod test { #[tokio::test] async fn test_config() { - let config = parse("pgcat.toml").await.unwrap(); - assert_eq!(config.general.pool_size, 15); - assert_eq!(config.shards.len(), 3); - assert_eq!(config.shards["1"].servers[0].0, "127.0.0.1"); - assert_eq!(config.shards["0"].servers[0].2, "primary"); - assert_eq!(config.query_router.default_role, "any"); + parse("pgcat.toml").await.unwrap(); + assert_eq!(get_config().general.pool_size, 15); + assert_eq!(get_config().shards.len(), 3); + assert_eq!(get_config().shards["1"].servers[0].0, "127.0.0.1"); + assert_eq!(get_config().shards["0"].servers[0].2, "primary"); + assert_eq!(get_config().query_router.default_role, "any"); } } diff --git a/src/main.rs b/src/main.rs index 1669331ad..a12137bec 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +extern crate arc_swap; extern crate async_trait; extern crate bb8; extern crate bytes; @@ -28,7 +29,10 @@ extern crate tokio; extern crate toml; use tokio::net::TcpListener; -use tokio::signal; +use tokio::{ + signal, + signal::unix::{signal as unix_signal, SignalKind}, +}; use std::collections::HashMap; use std::sync::{Arc, Mutex}; @@ -47,9 +51,8 @@ mod stats; // Support for query cancellation: this maps our process_ids and // secret keys to the backend's. -use config::Role; +use config::get_config; use pool::{ClientServerMap, ConnectionPool}; -use query_router::QueryRouter; use stats::{Collector, Reporter}; /// Main! @@ -63,14 +66,17 @@ async fn main() { return; } - let config = match config::parse("pgcat.toml").await { - Ok(config) => config, + // Prepare the config + match config::parse("pgcat.toml").await { + Ok(_) => (), Err(err) => { println!("> Config parse error: {:?}", err); return; } }; + let config = get_config(); + let addr = format!("{}:{}", config.general.host, config.general.port); let listener = match TcpListener::bind(&addr).await { Ok(sock) => sock, @@ -81,19 +87,11 @@ async fn main() { }; println!("> Running on {}", addr); + config.show(); // Tracks which client is connected to which server for query cancellation. let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); - println!("> Pool size: {}", config.general.pool_size); - println!("> Pool mode: {}", config.general.pool_mode); - println!("> Ban time: {}s", config.general.ban_time); - println!( - "> Healthcheck timeout: {}ms", - config.general.healthcheck_timeout - ); - println!("> Connection timeout: {}ms", config.general.connect_timeout); - // Collect statistics and send them to StatsD let (tx, rx) = mpsc::channel(100); @@ -104,25 +102,8 @@ async fn main() { stats_collector.collect().await; }); - let mut pool = ConnectionPool::from_config( - config.clone(), - client_server_map.clone(), - Reporter::new(tx.clone()), - ) - .await; - - let transaction_mode = config.general.pool_mode == "transaction"; - let default_server_role = match config.query_router.default_role.as_ref() { - "any" => None, - "primary" => Some(Role::Primary), - "replica" => Some(Role::Replica), - _ => { - println!("> Config error, got unexpected query_router.default_role."); - return; - } - }; - let primary_reads_enabled = config.query_router.primary_reads_enabled; - let query_parser_enabled = config.query_router.query_parser_enabled; + let mut pool = + ConnectionPool::from_config(client_server_map.clone(), Reporter::new(tx.clone())).await; let server_info = match pool.validate().await { Ok(info) => info, @@ -156,26 +137,13 @@ async fn main() { println!(">> Client {:?} connected", addr); - match client::Client::startup( - socket, - client_server_map, - transaction_mode, - server_info, - reporter, - ) - .await + match client::Client::startup(socket, client_server_map, server_info, reporter) + .await { Ok(mut client) => { println!(">> Client {:?} authenticated successfully!", addr); - let query_router = QueryRouter::new( - default_server_role, - pool.shards(), - primary_reads_enabled, - query_parser_enabled, - ); - - match client.handle(pool, query_router).await { + match client.handle(pool).await { Ok(()) => { let duration = chrono::offset::Utc::now().naive_utc() - start; @@ -201,6 +169,26 @@ async fn main() { } }); + // Reload config + // kill -SIGHUP $(pgrep pgcat) + tokio::task::spawn(async move { + let mut stream = unix_signal(SignalKind::hangup()).unwrap(); + + loop { + stream.recv().await; + println!("> Reloading config"); + match config::parse("pgcat.toml").await { + Ok(_) => { + get_config().show(); + } + Err(err) => { + println!("> Config parse error: {:?}", err); + return; + } + }; + } + }); + // Setup shut down sequence match signal::ctrl_c().await { Ok(()) => { diff --git a/src/messages.rs b/src/messages.rs index 70dbc10e3..16b2f84c3 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -38,6 +38,7 @@ pub async fn backend_key_data( Ok(write_all(stream, key_data).await?) } +#[allow(dead_code)] pub fn simple_query(query: &str) -> BytesMut { let mut res = BytesMut::from(&b"Q"[..]); let query = format!("{}\0", query); diff --git a/src/pool.rs b/src/pool.rs index 6a190f11e..29ee40ba7 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -4,7 +4,7 @@ use bb8::{ManageConnection, Pool, PooledConnection}; use bytes::BytesMut; use chrono::naive::NaiveDateTime; -use crate::config::{Address, Config, Role, User}; +use crate::config::{get_config, Address, Role, User}; use crate::errors::Error; use crate::server::Server; use crate::stats::Reporter; @@ -23,18 +23,16 @@ pub struct ConnectionPool { addresses: Vec>, round_robin: usize, banlist: BanList, - healthcheck_timeout: u64, - ban_time: i64, stats: Reporter, } impl ConnectionPool { /// Construct the connection pool from a config file. pub async fn from_config( - config: Config, client_server_map: ClientServerMap, stats: Reporter, ) -> ConnectionPool { + let config = get_config(); let mut shards = Vec::new(); let mut addresses = Vec::new(); let mut banlist = Vec::new(); @@ -103,8 +101,6 @@ impl ConnectionPool { addresses: addresses, round_robin: rand::random::() % address_len, // Start at a random replica banlist: Arc::new(Mutex::new(banlist)), - healthcheck_timeout: config.general.healthcheck_timeout, - ban_time: config.general.ban_time, stats: stats, } } @@ -214,9 +210,10 @@ impl ConnectionPool { // // Check if this server is alive with a health check let server = &mut *conn; + let healthcheck_timeout = get_config().general.healthcheck_timeout; match tokio::time::timeout( - tokio::time::Duration::from_millis(self.healthcheck_timeout), + tokio::time::Duration::from_millis(healthcheck_timeout), server.query("SELECT 1"), ) .await @@ -303,8 +300,9 @@ impl ConnectionPool { match guard[shard].get(address) { Some(timestamp) => { let now = chrono::offset::Utc::now().naive_utc(); + let config = get_config(); // Ban expired. - if now.timestamp() - timestamp.timestamp() > self.ban_time { + if now.timestamp() - timestamp.timestamp() > config.general.ban_time { guard[shard].remove(address); false } else { diff --git a/src/query_router.rs b/src/query_router.rs index c43ca4973..097cdefe9 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1,4 +1,4 @@ -use crate::config::Role; +use crate::config::{get_config, Role}; use crate::sharding::Sharder; /// Route queries automatically based on explicitely requested /// or implied query characteristics. @@ -65,20 +65,24 @@ impl QueryRouter { } } - pub fn new( - default_server_role: Option, - shards: usize, - primary_reads_enabled: bool, - query_parser_enabled: bool, - ) -> QueryRouter { + pub fn new() -> QueryRouter { + let config = get_config(); + + let default_server_role = match config.query_router.default_role.as_ref() { + "any" => None, + "primary" => Some(Role::Primary), + "replica" => Some(Role::Replica), + _ => unreachable!(), + }; + QueryRouter { default_server_role: default_server_role, - shards: shards, + shards: config.shards.len(), active_role: default_server_role, active_shard: None, - primary_reads_enabled: primary_reads_enabled, - query_parser_enabled: query_parser_enabled, + primary_reads_enabled: config.query_router.primary_reads_enabled, + query_parser_enabled: config.query_router.query_parser_enabled, } } @@ -275,6 +279,11 @@ impl QueryRouter { pub fn query_parser_enabled(&self) -> bool { self.query_parser_enabled } + + #[allow(dead_code)] + pub fn toggle_primary_reads(&mut self, value: bool) { + self.primary_reads_enabled = value; + } } #[cfg(test)] @@ -286,10 +295,7 @@ mod test { #[test] fn test_defaults() { QueryRouter::setup(); - - let default_server_role: Option = None; - let shards = 5; - let qr = QueryRouter::new(default_server_role, shards, false, false); + let qr = QueryRouter::new(); assert_eq!(qr.role(), None); } @@ -297,10 +303,10 @@ mod test { #[test] fn test_infer_role_replica() { QueryRouter::setup(); - - let default_server_role: Option = None; - let shards = 5; - let mut qr = QueryRouter::new(default_server_role, shards, false, false); + let mut qr = QueryRouter::new(); + assert!(qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")) != None); + assert_eq!(qr.query_parser_enabled(), true); + qr.toggle_primary_reads(false); let queries = vec![ simple_query("SELECT * FROM items WHERE id = 5"), @@ -320,10 +326,7 @@ mod test { #[test] fn test_infer_role_primary() { QueryRouter::setup(); - - let default_server_role: Option = None; - let shards = 5; - let mut qr = QueryRouter::new(default_server_role, shards, false, false); + let mut qr = QueryRouter::new(); let queries = vec![ simple_query("UPDATE items SET name = 'pumpkin' WHERE id = 5"), @@ -342,11 +345,9 @@ mod test { #[test] fn test_infer_role_primary_reads_enabled() { QueryRouter::setup(); - - let default_server_role: Option = None; - let shards = 5; - let mut qr = QueryRouter::new(default_server_role, shards, true, false); + let mut qr = QueryRouter::new(); let query = simple_query("SELECT * FROM items WHERE id = 5"); + qr.toggle_primary_reads(true); assert!(qr.infer_role(query)); assert_eq!(qr.role(), None); @@ -355,11 +356,9 @@ mod test { #[test] fn test_infer_role_parse_prepared() { QueryRouter::setup(); - - let default_server_role: Option = None; - let shards = 5; - - let mut query_router = QueryRouter::new(default_server_role, shards, false, false); + let mut qr = QueryRouter::new(); + qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")); + qr.toggle_primary_reads(false); let prepared_stmt = BytesMut::from( &b"WITH t AS (SELECT * FROM items WHERE name = $1) SELECT * FROM t WHERE id = $2\0"[..], @@ -370,8 +369,8 @@ mod test { res.put(prepared_stmt); res.put_i16(0); - assert!(query_router.infer_role(res)); - assert_eq!(query_router.role(), Some(Role::Replica)); + assert!(qr.infer_role(res)); + assert_eq!(qr.role(), Some(Role::Replica)); } #[test] @@ -411,15 +410,15 @@ mod test { #[test] fn test_try_execute_command() { QueryRouter::setup(); - let mut qr = QueryRouter::new(Some(Role::Primary), 5, false, false); + let mut qr = QueryRouter::new(); // SetShardingKey let query = simple_query("SET SHARDING KEY TO '13'"); assert_eq!( qr.try_execute_command(query), - Some((Command::SetShardingKey, String::from("3"))) + Some((Command::SetShardingKey, String::from("1"))) ); - assert_eq!(qr.shard(), 3); + assert_eq!(qr.shard(), 1); // SetShard let query = simple_query("SET SHARD TO '1'"); @@ -468,8 +467,9 @@ mod test { #[test] fn test_enable_query_parser() { QueryRouter::setup(); - let mut qr = QueryRouter::new(None, 5, false, false); + let mut qr = QueryRouter::new(); let query = simple_query("SET SERVER ROLE TO 'auto'"); + qr.toggle_primary_reads(false); assert!(qr.try_execute_command(query) != None); assert!(qr.query_parser_enabled()); diff --git a/src/stats.rs b/src/stats.rs index 3f93b3f2b..f3e079071 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -5,6 +5,8 @@ use tokio::sync::mpsc::{Receiver, Sender}; use std::collections::HashMap; use std::time::Instant; +use crate::config::get_config; + #[derive(Debug)] pub enum StatisticName { CheckoutTime, @@ -138,7 +140,7 @@ impl Collector { pub fn new(rx: Receiver) -> Collector { Collector { rx: rx, - client: Client::new("127.0.0.1:8125", "pgcat").unwrap(), + client: Client::new(&get_config().general.statsd_address, "pgcat").unwrap(), } } From cb8bf56e7c9478dc89184073f6e0c13d0e83c222 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 20 Feb 2022 12:40:09 -0800 Subject: [PATCH 094/551] Fix client states reporting (#27) * Fix client states reporting * clean --- src/client.rs | 18 ++++++++-- src/pool.rs | 5 --- src/stats.rs | 97 +++++++++++++++++++++++++++++---------------------- 3 files changed, 70 insertions(+), 50 deletions(-) diff --git a/src/client.rs b/src/client.rs index 1b921b84b..7a11d60c4 100644 --- a/src/client.rs +++ b/src/client.rs @@ -192,6 +192,9 @@ impl Client { // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocols. loop { + // Client idle, waiting for messages. + self.stats.client_idle(self.process_id); + // Read a complete message from the client, which normally would be // either a `Q` (query) or `P` (prepare, extended protocol). // We can parse it here before grabbing a server from the pool, @@ -243,6 +246,9 @@ impl Client { continue; } + // Waiting for server connection. + self.stats.client_waiting(self.process_id); + // Grab a server from the pool: the client issued a regular query. let connection = match pool.get(query_router.shard(), query_router.role()).await { Ok(conn) => conn, @@ -261,6 +267,9 @@ impl Client { // Claim this server as mine for query cancellation. server.claim(self.process_id, self.secret_key); + // Client active + self.stats.client_active(self.process_id); + // Transaction loop. Multiple queries can be issued by the client here. // The connection belongs to the client until the transaction is over, // or until the client disconnects if we are in session mode. @@ -330,7 +339,6 @@ impl Client { // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { // Report this client as idle. - self.stats.client_idle(); break; } } @@ -412,7 +420,6 @@ impl Client { self.stats.transaction(); if self.transaction_mode { - self.stats.client_idle(); break; } } @@ -446,7 +453,6 @@ impl Client { self.stats.transaction(); if self.transaction_mode { - self.stats.client_idle(); break; } } @@ -471,3 +477,9 @@ impl Client { guard.remove(&(self.process_id, self.secret_key)); } } + +impl Drop for Client { + fn drop(&mut self) { + self.stats.client_disconnecting(self.process_id); + } +} diff --git a/src/pool.rs b/src/pool.rs index 29ee40ba7..bea260bdc 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -147,10 +147,6 @@ impl ConnectionPool { role: Option, ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { let now = Instant::now(); - - // We are waiting for a server now. - self.stats.client_waiting(); - let addresses = &self.addresses[shard]; let mut allowed_attempts = match role { @@ -222,7 +218,6 @@ impl ConnectionPool { Ok(res) => match res { Ok(_) => { self.stats.checkout_time(now.elapsed().as_micros()); - self.stats.client_active(); return Ok((conn, address.clone())); } Err(_) => { diff --git a/src/stats.rs b/src/stats.rs index f3e079071..e74c0685f 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -7,7 +7,7 @@ use std::time::Instant; use crate::config::get_config; -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub enum StatisticName { CheckoutTime, //QueryRuntime, @@ -16,15 +16,17 @@ pub enum StatisticName { Transactions, DataSent, DataReceived, - ClientsWaiting, - ClientsActive, - ClientsIdle, + ClientWaiting, + ClientActive, + ClientIdle, + ClientDisconnecting, } #[derive(Debug)] pub struct Statistic { pub name: StatisticName, pub value: i64, + pub process_id: Option, } #[derive(Clone, Debug)] @@ -41,6 +43,7 @@ impl Reporter { let statistic = Statistic { name: StatisticName::Queries, value: 1, + process_id: None, }; let _ = self.tx.try_send(statistic); @@ -50,6 +53,7 @@ impl Reporter { let statistic = Statistic { name: StatisticName::Transactions, value: 1, + process_id: None, }; let _ = self.tx.try_send(statistic); @@ -59,6 +63,7 @@ impl Reporter { let statistic = Statistic { name: StatisticName::DataSent, value: amount as i64, + process_id: None, }; let _ = self.tx.try_send(statistic); @@ -68,6 +73,7 @@ impl Reporter { let statistic = Statistic { name: StatisticName::DataReceived, value: amount as i64, + process_id: None, }; let _ = self.tx.try_send(statistic); @@ -77,54 +83,48 @@ impl Reporter { let statistic = Statistic { name: StatisticName::CheckoutTime, value: ms as i64, + process_id: None, }; let _ = self.tx.try_send(statistic); } - pub fn client_waiting(&mut self) { + pub fn client_waiting(&mut self, process_id: i32) { let statistic = Statistic { - name: StatisticName::ClientsWaiting, + name: StatisticName::ClientWaiting, value: 1, - }; - - let _ = self.tx.try_send(statistic); - - let statistic = Statistic { - name: StatisticName::ClientsIdle, - value: -1, + process_id: Some(process_id), }; let _ = self.tx.try_send(statistic); } - pub fn client_active(&mut self) { - let statistic = Statistic { - name: StatisticName::ClientsWaiting, - value: -1, - }; - - let _ = self.tx.try_send(statistic); + pub fn client_active(&mut self, process_id: i32) { let statistic = Statistic { - name: StatisticName::ClientsActive, + name: StatisticName::ClientActive, value: 1, + process_id: Some(process_id), }; let _ = self.tx.try_send(statistic); } - pub fn client_idle(&mut self) { + pub fn client_idle(&mut self, process_id: i32) { let statistic = Statistic { - name: StatisticName::ClientsActive, - value: -1, + name: StatisticName::ClientIdle, + value: 1, + process_id: Some(process_id), }; let _ = self.tx.try_send(statistic); + } + pub fn client_disconnecting(&mut self, process_id: i32) { let statistic = Statistic { - name: StatisticName::ClientsIdle, + name: StatisticName::ClientDisconnecting, value: 1, + process_id: Some(process_id), }; let _ = self.tx.try_send(statistic); @@ -158,6 +158,8 @@ impl Collector { ("cl_idle", 0), ]); + let mut client_states: HashMap = HashMap::new(); + let mut now = Instant::now(); loop { @@ -210,32 +212,43 @@ impl Collector { } } - StatisticName::ClientsActive => { - let counter = stats.entry("cl_active").or_insert(0); - - *counter += stat.value; - *counter = std::cmp::max(*counter, 0); - } - - StatisticName::ClientsWaiting => { - let counter = stats.entry("cl_waiting").or_insert(0); - *counter += stat.value; - *counter = std::cmp::max(*counter, 0); + StatisticName::ClientActive | StatisticName::ClientWaiting | StatisticName::ClientIdle => { + client_states.insert(stat.process_id.unwrap(), stat.name); } - StatisticName::ClientsIdle => { - let counter = stats.entry("cl_idle").or_insert(0); - *counter += stat.value; - *counter = std::cmp::max(*counter, 0); + StatisticName::ClientDisconnecting => { + client_states.remove(&stat.process_id.unwrap()); } }; + // It's been 15 seconds. If there is no traffic, it won't publish anything, // but it also doesn't matter then. if now.elapsed().as_secs() > 15 { - let mut pipeline = self.client.pipeline(); + for (_, state) in &client_states { + match state { + StatisticName::ClientActive => { + let counter = stats.entry("cl_active").or_insert(0); + *counter += 1; + } + + StatisticName::ClientWaiting => { + let counter = stats.entry("cl_waiting").or_insert(0); + *counter += 1; + } + + StatisticName::ClientIdle => { + let counter = stats.entry("cl_idle").or_insert(0); + *counter += 1; + } + + _ => unreachable!(), + }; + } + + println!(">> Reporting to StatsD: {:?}", stats); - println!(">> Publishing statistics to StatsD: {:?}", stats); + let mut pipeline = self.client.pipeline(); for (key, value) in stats.iter_mut() { pipeline.gauge(key, *value as f64); From 64cc360551497a6e273c75e1551ebafbf6e3b6a4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 20 Feb 2022 13:49:30 -0800 Subject: [PATCH 095/551] License as MIT (#28) * License as MIT * license * s --- LICENSE | 694 ++-------------------------------------------------- src/main.rs | 34 +-- 2 files changed, 40 insertions(+), 688 deletions(-) diff --git a/LICENSE b/LICENSE index 94a9ed024..5cd126175 100644 --- a/LICENSE +++ b/LICENSE @@ -1,674 +1,20 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. +Copyright (c) 2022 Lev Kokotov + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/main.rs b/src/main.rs index a12137bec..e03cd12a6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,18 +1,24 @@ -// PgCat, a PostgreSQL pooler with load balancing, failover, and sharding support. -// Copyright (C) 2022 Lev Kokotov +// Copyright (c) 2022 Lev Kokotov + +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: + +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . extern crate arc_swap; extern crate async_trait; extern crate bb8; From 37088dfc9154921036e897500f9499f6a4bfa753 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 20 Feb 2022 22:47:08 -0800 Subject: [PATCH 096/551] use logger lib; minor refactor; sv_* stats (#29) --- Cargo.lock | 49 ++++++++++++++++++++ Cargo.toml | 1 + src/client.rs | 12 +++-- src/config.rs | 33 +++++++------- src/main.rs | 45 +++++++++---------- src/pool.rs | 33 +++++++------- src/query_router.rs | 7 +-- src/server.rs | 39 +++++++++------- src/stats.rs | 106 +++++++++++++++++++++++++++++++++++++++----- 9 files changed, 236 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index befc8f137..1259b4e94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,17 @@ dependencies = [ "syn", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.0.1" @@ -116,6 +127,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "env_logger" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "futures-channel" version = "0.3.19" @@ -181,6 +205,12 @@ dependencies = [ "libc", ] +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "instant" version = "0.1.12" @@ -329,6 +359,7 @@ dependencies = [ "bb8", "bytes", "chrono", + "env_logger", "log", "md-5", "num_cpus", @@ -530,6 +561,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + [[package]] name = "time" version = "0.1.44" @@ -620,6 +660,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 7afcc6a8c..860955e93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,3 +24,4 @@ statsd = "0.15" sqlparser = "0.14" log = "0.4" arc-swap = "1" +env_logger = "0.9" diff --git a/src/client.rs b/src/client.rs index 7a11d60c4..06f5a9f65 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,6 +2,7 @@ /// We are pretending to the server in this scenario, /// and this module implements that. use bytes::{Buf, BufMut, BytesMut}; +use log::error; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, @@ -253,7 +254,7 @@ impl Client { let connection = match pool.get(query_router.shard(), query_router.role()).await { Ok(conn) => conn, Err(err) => { - println!(">> Could not get connection from pool: {:?}", err); + error!("Could not get connection from pool: {:?}", err); error_response(&mut self.write, "could not get connection from the pool") .await?; continue; @@ -267,8 +268,9 @@ impl Client { // Claim this server as mine for query cancellation. server.claim(self.process_id, self.secret_key); - // Client active + // Client active & server active self.stats.client_active(self.process_id); + self.stats.server_active(server.process_id()); // Transaction loop. Multiple queries can be issued by the client here. // The connection belongs to the client until the transaction is over, @@ -338,7 +340,7 @@ impl Client { // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { - // Report this client as idle. + self.stats.server_idle(server.process_id()); break; } } @@ -420,6 +422,7 @@ impl Client { self.stats.transaction(); if self.transaction_mode { + self.stats.server_idle(server.process_id()); break; } } @@ -453,6 +456,7 @@ impl Client { self.stats.transaction(); if self.transaction_mode { + self.stats.server_idle(server.process_id()); break; } } @@ -461,7 +465,7 @@ impl Client { // Some unexpected message. We either did not implement the protocol correctly // or this is not a Postgres client we're talking to. _ => { - println!(">>> Unexpected code: {}", code); + error!("Unexpected code: {}", code); } } } diff --git a/src/config.rs b/src/config.rs index 261d596f8..221fb6673 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,5 @@ use arc_swap::{ArcSwap, Guard}; +use log::{error, info}; use once_cell::sync::Lazy; use serde_derive::Deserialize; use tokio::fs::File; @@ -150,14 +151,14 @@ impl Default for Config { impl Config { pub fn show(&self) { - println!("> Pool size: {}", self.general.pool_size); - println!("> Pool mode: {}", self.general.pool_mode); - println!("> Ban time: {}s", self.general.ban_time); - println!( - "> Healthcheck timeout: {}ms", + info!("Pool size: {}", self.general.pool_size); + info!("Pool mode: {}", self.general.pool_mode); + info!("Ban time: {}s", self.general.ban_time); + info!( + "Healthcheck timeout: {}ms", self.general.healthcheck_timeout ); - println!("> Connection timeout: {}ms", self.general.connect_timeout); + info!("Connection timeout: {}ms", self.general.connect_timeout); } } @@ -171,7 +172,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { let mut file = match File::open(path).await { Ok(file) => file, Err(err) => { - println!("> Config error: {:?}", err); + error!("{:?}", err); return Err(Error::BadConfig); } }; @@ -179,7 +180,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { match file.read_to_string(&mut contents).await { Ok(_) => (), Err(err) => { - println!("> Config error: {:?}", err); + error!("{:?}", err); return Err(Error::BadConfig); } }; @@ -187,7 +188,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { let config: Config = match toml::from_str(&contents) { Ok(config) => config, Err(err) => { - println!("> Config error: {:?}", err); + error!("{:?}", err); return Err(Error::BadConfig); } }; @@ -200,7 +201,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { let mut primary_count = 0; if shard.1.servers.len() == 0 { - println!("> Shard {} has no servers configured", shard.0); + error!("Shard {} has no servers configured", shard.0); return Err(Error::BadConfig); } @@ -218,8 +219,8 @@ pub async fn parse(path: &str) -> Result<(), Error> { "primary" => (), "replica" => (), _ => { - println!( - "> Shard {} server role must be either 'primary' or 'replica', got: '{}'", + error!( + "Shard {} server role must be either 'primary' or 'replica', got: '{}'", shard.0, server.2 ); return Err(Error::BadConfig); @@ -228,12 +229,12 @@ pub async fn parse(path: &str) -> Result<(), Error> { } if primary_count > 1 { - println!("> Shard {} has more than on primary configured.", &shard.0); + error!("Shard {} has more than on primary configured", &shard.0); return Err(Error::BadConfig); } if dup_check.len() != shard.1.servers.len() { - println!("> Shard {} contains duplicate server configs.", &shard.0); + error!("Shard {} contains duplicate server configs", &shard.0); return Err(Error::BadConfig); } } @@ -243,8 +244,8 @@ pub async fn parse(path: &str) -> Result<(), Error> { "primary" => (), "replica" => (), other => { - println!( - "> Query router default_role must be 'primary', 'replica', or 'any', got: '{}'", + error!( + "Query router default_role must be 'primary', 'replica', or 'any', got: '{}'", other ); return Err(Error::BadConfig); diff --git a/src/main.rs b/src/main.rs index e03cd12a6..65a12c9a3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,7 @@ extern crate arc_swap; extern crate async_trait; extern crate bb8; extern crate bytes; +extern crate env_logger; extern crate log; extern crate md5; extern crate num_cpus; @@ -34,15 +35,16 @@ extern crate statsd; extern crate tokio; extern crate toml; +use log::{error, info}; use tokio::net::TcpListener; use tokio::{ signal, signal::unix::{signal as unix_signal, SignalKind}, + sync::mpsc, }; use std::collections::HashMap; use std::sync::{Arc, Mutex}; -use tokio::sync::mpsc; mod client; mod config; @@ -64,11 +66,12 @@ use stats::{Collector, Reporter}; /// Main! #[tokio::main(worker_threads = 4)] async fn main() { - println!("> Welcome to PgCat! Meow."); + env_logger::init(); + info!("Welcome to PgCat! Meow."); // Prepare regexes if !query_router::QueryRouter::setup() { - println!("> Could not setup query router."); + error!("Could not setup query router"); return; } @@ -76,7 +79,7 @@ async fn main() { match config::parse("pgcat.toml").await { Ok(_) => (), Err(err) => { - println!("> Config parse error: {:?}", err); + error!("Config parse error: {:?}", err); return; } }; @@ -87,12 +90,12 @@ async fn main() { let listener = match TcpListener::bind(&addr).await { Ok(sock) => sock, Err(err) => { - println!("> Error: {:?}", err); + error!("Listener socket error: {:?}", err); return; } }; - println!("> Running on {}", addr); + info!("Running on {}", addr); config.show(); // Tracks which client is connected to which server for query cancellation. @@ -102,8 +105,6 @@ async fn main() { let (tx, rx) = mpsc::channel(100); tokio::task::spawn(async move { - println!("> Statistics reporter started"); - let mut stats_collector = Collector::new(rx); stats_collector.collect().await; }); @@ -114,12 +115,12 @@ async fn main() { let server_info = match pool.validate().await { Ok(info) => info, Err(err) => { - println!("> Could not validate connection pool: {:?}", err); + error!("Could not validate connection pool: {:?}", err); return; } }; - println!("> Waiting for clients..."); + info!("Waiting for clients"); // Main app runs here. tokio::task::spawn(async move { @@ -132,7 +133,7 @@ async fn main() { let (socket, addr) = match listener.accept().await { Ok((socket, addr)) => (socket, addr), Err(err) => { - println!("> Listener: {:?}", err); + error!("{:?}", err); continue; } }; @@ -140,35 +141,31 @@ async fn main() { // Client goes to another thread, bye. tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); - - println!(">> Client {:?} connected", addr); - match client::Client::startup(socket, client_server_map, server_info, reporter) .await { Ok(mut client) => { - println!(">> Client {:?} authenticated successfully!", addr); - + info!("Client {:?} connected", addr); match client.handle(pool).await { Ok(()) => { let duration = chrono::offset::Utc::now().naive_utc() - start; - println!( - ">> Client {:?} disconnected, session duration: {}", + info!( + "Client {:?} disconnected, session duration: {}", addr, format_duration(&duration) ); } Err(err) => { - println!(">> Client disconnected with error: {:?}", err); + error!("Client disconnected with error: {:?}", err); client.release(); } } } Err(err) => { - println!(">> Error: {:?}", err); + error!("Client failed to login: {:?}", err); } }; }); @@ -182,13 +179,13 @@ async fn main() { loop { stream.recv().await; - println!("> Reloading config"); + info!("Reloading config"); match config::parse("pgcat.toml").await { Ok(_) => { get_config().show(); } Err(err) => { - println!("> Config parse error: {:?}", err); + error!("{:?}", err); return; } }; @@ -198,11 +195,11 @@ async fn main() { // Setup shut down sequence match signal::ctrl_c().await { Ok(()) => { - println!("> Shutting down..."); + info!("Shutting down..."); } Err(err) => { - eprintln!("Unable to listen for shutdown signal: {}", err); + error!("Unable to listen for shutdown signal: {}", err); } }; } diff --git a/src/pool.rs b/src/pool.rs index bea260bdc..87f169a3e 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -3,6 +3,7 @@ use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection}; use bytes::BytesMut; use chrono::naive::NaiveDateTime; +use log::{error, info, warn}; use crate::config::{get_config, Address, Role, User}; use crate::errors::Error; @@ -54,7 +55,7 @@ impl ConnectionPool { "primary" => Role::Primary, "replica" => Role::Replica, _ => { - println!("> Config error: server role can be 'primary' or 'replica', have: '{}'. Defaulting to 'replica'.", server.2); + error!("Config error: server role can be 'primary' or 'replica', have: '{}'. Defaulting to 'replica'.", server.2); Role::Replica } }; @@ -118,7 +119,7 @@ impl ConnectionPool { let connection = match self.get(shard, None).await { Ok(conn) => conn, Err(err) => { - println!("> Shard {} down or misconfigured: {:?}", shard, err); + error!("Shard {} down or misconfigured: {:?}", shard, err); continue; } }; @@ -166,8 +167,8 @@ impl ConnectionPool { }; if !exists { - log::error!( - "ConnectionPool::get Requested role {:?}, but none is configured.", + error!( + "Requested role {:?}, but none are configured", role ); return Err(Error::BadConfig); @@ -198,7 +199,7 @@ impl ConnectionPool { let mut conn = match self.databases[shard][index].get().await { Ok(conn) => conn, Err(err) => { - println!(">> Banning replica {}, error: {:?}", index, err); + error!("Banning replica {}, error: {:?}", index, err); self.ban(address, shard); continue; } @@ -208,6 +209,8 @@ impl ConnectionPool { let server = &mut *conn; let healthcheck_timeout = get_config().general.healthcheck_timeout; + self.stats.server_tested(server.process_id()); + match tokio::time::timeout( tokio::time::Duration::from_millis(healthcheck_timeout), server.query("SELECT 1"), @@ -218,13 +221,11 @@ impl ConnectionPool { Ok(res) => match res { Ok(_) => { self.stats.checkout_time(now.elapsed().as_micros()); + self.stats.server_idle(conn.process_id()); return Ok((conn, address.clone())); } Err(_) => { - println!( - ">> Banning replica {} because of failed health check", - index - ); + error!("Banning replica {} because of failed health check", index); // Don't leave a bad connection in the pool. server.mark_bad(); @@ -234,10 +235,7 @@ impl ConnectionPool { }, // Health check never came back, database is really really down Err(_) => { - println!( - ">> Banning replica {} because of health check timeout", - index - ); + error!("Banning replica {} because of health check timeout", index); // Don't leave a bad connection in the pool. server.mark_bad(); @@ -254,7 +252,7 @@ impl ConnectionPool { /// traffic for any new transactions. Existing transactions on that replica /// will finish successfully or error out to the clients. pub fn ban(&self, address: &Address, shard: usize) { - println!(">> Banning {:?}", address); + error!("Banning {:?}", address); let now = chrono::offset::Utc::now().naive_utc(); let mut guard = self.banlist.lock().unwrap(); guard[shard].insert(address.clone(), now); @@ -287,7 +285,7 @@ impl ConnectionPool { if guard[shard].len() == replicas_available { guard[shard].clear(); drop(guard); - println!(">> Unbanning all replicas."); + warn!("Unbanning all replicas."); return false; } @@ -351,7 +349,10 @@ impl ManageConnection for ServerPool { /// Attempts to create a new connection. async fn connect(&self) -> Result { - println!(">> Creating a new connection for the pool"); + info!( + "Creating a new connection to {:?} using user {:?}", + self.address, self.user.name + ); Server::startup( &self.address, diff --git a/src/query_router.rs b/src/query_router.rs index 097cdefe9..64d6e7042 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -8,6 +8,7 @@ use regex::RegexSet; use sqlparser::ast::Statement::{Query, StartTransaction}; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; +use log::{error, debug}; const CUSTOM_SQL_REGEXES: [&str; 5] = [ r"(?i)SET SHARDING KEY TO '[0-9]+'", @@ -54,7 +55,7 @@ impl QueryRouter { let set = match RegexSet::new(&CUSTOM_SQL_REGEXES) { Ok(rgx) => rgx, Err(err) => { - log::error!("QueryRouter::setup Could not compile regex set: {:?}", err); + error!("QueryRouter::setup Could not compile regex set: {:?}", err); return false; } }; @@ -219,8 +220,8 @@ impl QueryRouter { let ast = match Parser::parse_sql(&PostgreSqlDialect {}, &query) { Ok(ast) => ast, Err(err) => { - log::debug!( - "QueryParser::infer_role could not parse query, error: {:?}, query: {}", + debug!( + "{:?}, query: {}", err, query ); diff --git a/src/server.rs b/src/server.rs index 842b81fc7..18b1f267e 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,6 +1,7 @@ +use bytes::{Buf, BufMut, BytesMut}; ///! Implementation of the PostgreSQL server (database) protocol. ///! Here we are pretending to the a Postgres client. -use bytes::{Buf, BufMut, BytesMut}; +use log::{error, info}; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, @@ -33,7 +34,7 @@ pub struct Server { server_info: BytesMut, // Backend id and secret key used for query cancellation. - backend_id: i32, + process_id: i32, secret_key: i32, // Is the server inside a transaction or idle. @@ -69,7 +70,7 @@ impl Server { match TcpStream::connect(&format!("{}:{}", &address.host, &address.port)).await { Ok(stream) => stream, Err(err) => { - println!(">> Could not connect to server: {}", err); + error!("Could not connect to server: {}", err); return Err(Error::SocketError); } }; @@ -78,7 +79,7 @@ impl Server { startup(&mut stream, &user.name, database).await?; let mut server_info = BytesMut::new(); - let mut backend_id: i32 = 0; + let mut process_id: i32 = 0; let mut secret_key: i32 = 0; // We'll be handling multiple packets, but they will all be structured the same. @@ -121,7 +122,7 @@ impl Server { AUTHENTICATION_SUCCESSFUL => (), _ => { - println!(">> Unsupported authentication mechanism: {}", auth_code); + error!("Unsupported authentication mechanism: {}", auth_code); return Err(Error::ServerError); } } @@ -151,7 +152,7 @@ impl Server { // TODO: the error message contains multiple fields; we can decode them and // present a prettier message to the user. // See: https://www.postgresql.org/docs/12/protocol-error-fields.html - println!(">> Server error: {}", String::from_utf8_lossy(&error)); + error!("Server error: {}", String::from_utf8_lossy(&error)); } }; @@ -179,7 +180,7 @@ impl Server { 'K' => { // The frontend must save these values if it wishes to be able to issue CancelRequest messages later. // See: https://www.postgresql.org/docs/12/protocol-message-formats.html - backend_id = match stream.read_i32().await { + process_id = match stream.read_i32().await { Ok(id) => id, Err(_) => return Err(Error::SocketError), }; @@ -209,7 +210,7 @@ impl Server { write: write, buffer: BytesMut::with_capacity(8196), server_info: server_info, - backend_id: backend_id, + process_id: process_id, secret_key: secret_key, in_transaction: false, data_available: false, @@ -223,7 +224,7 @@ impl Server { // We have an unexpected message from the server during this exchange. // Means we implemented the protocol wrong or we're not talking to a Postgres server. _ => { - println!(">> Unknown code: {}", code); + error!("Unknown code: {}", code); return Err(Error::ProtocolSyncError); } }; @@ -241,7 +242,7 @@ impl Server { let mut stream = match TcpStream::connect(&format!("{}:{}", host, port)).await { Ok(stream) => stream, Err(err) => { - println!(">> Could not connect to server: {}", err); + error!("Could not connect to server: {}", err); return Err(Error::SocketError); } }; @@ -262,7 +263,7 @@ impl Server { match write_all_half(&mut self.write, messages).await { Ok(_) => Ok(()), Err(err) => { - println!(">> Terminating server because of: {:?}", err); + error!("Terminating server because of: {:?}", err); self.bad = true; Err(err) } @@ -277,7 +278,7 @@ impl Server { let mut message = match read_message(&mut self.read).await { Ok(message) => message, Err(err) => { - println!(">> Terminating server because of: {:?}", err); + error!("Terminating server because of: {:?}", err); self.bad = true; return Err(err); } @@ -396,7 +397,7 @@ impl Server { /// Indicate that this server connection cannot be re-used and must be discarded. pub fn mark_bad(&mut self) { - println!(">> Server marked bad"); + error!("Server marked bad"); self.bad = true; } @@ -406,7 +407,7 @@ impl Server { guard.insert( (process_id, secret_key), ( - self.backend_id, + self.process_id, self.secret_key, self.address.host.clone(), self.address.port.clone(), @@ -455,6 +456,10 @@ impl Server { pub fn address(&self) -> Address { self.address.clone() } + + pub fn process_id(&self) -> i32 { + self.process_id + } } impl Drop for Server { @@ -462,6 +467,8 @@ impl Drop for Server { /// the socket is in non-blocking mode, so it may not be ready /// for a write. fn drop(&mut self) { + self.stats.server_disconnecting(self.process_id()); + let mut bytes = BytesMut::with_capacity(4); bytes.put_u8(b'X'); bytes.put_i32(4); @@ -476,8 +483,8 @@ impl Drop for Server { let now = chrono::offset::Utc::now().naive_utc(); let duration = now - self.connected_at; - println!( - ">> Server connection closed, session duration: {}", + info!( + "Server connection closed, session duration: {}", crate::format_duration(&duration) ); } diff --git a/src/stats.rs b/src/stats.rs index e74c0685f..beb430f11 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,3 +1,4 @@ +use log::info; use statsd::Client; /// Statistics collector and publisher. use tokio::sync::mpsc::{Receiver, Sender}; @@ -20,6 +21,11 @@ pub enum StatisticName { ClientActive, ClientIdle, ClientDisconnecting, + ServerActive, + ServerIdle, + ServerTested, + ServerLogin, + ServerDisconnecting, } #[derive(Debug)] @@ -100,7 +106,6 @@ impl Reporter { } pub fn client_active(&mut self, process_id: i32) { - let statistic = Statistic { name: StatisticName::ClientActive, value: 1, @@ -129,6 +134,56 @@ impl Reporter { let _ = self.tx.try_send(statistic); } + + pub fn server_active(&mut self, process_id: i32) { + let statistic = Statistic { + name: StatisticName::ServerActive, + value: 1, + process_id: Some(process_id), + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn server_idle(&mut self, process_id: i32) { + let statistic = Statistic { + name: StatisticName::ServerIdle, + value: 1, + process_id: Some(process_id), + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn server_login(&mut self, process_id: i32) { + let statistic = Statistic { + name: StatisticName::ServerLogin, + value: 1, + process_id: Some(process_id), + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn server_tested(&mut self, process_id: i32) { + let statistic = Statistic { + name: StatisticName::ServerTested, + value: 1, + process_id: Some(process_id), + }; + + let _ = self.tx.try_send(statistic); + } + + pub fn server_disconnecting(&mut self, process_id: i32) { + let statistic = Statistic { + name: StatisticName::ServerDisconnecting, + value: 1, + process_id: Some(process_id), + }; + + let _ = self.tx.try_send(statistic); + } } pub struct Collector { @@ -145,6 +200,8 @@ impl Collector { } pub async fn collect(&mut self) { + info!("Statistics reporter started"); + let mut stats = HashMap::from([ ("total_query_count", 0), ("total_xact_count", 0), @@ -156,9 +213,13 @@ impl Collector { ("cl_waiting", 0), ("cl_active", 0), ("cl_idle", 0), + ("sv_idle", 0), + ("sv_active", 0), + ("sv_login", 0), + ("sv_tested", 0), ]); - let mut client_states: HashMap = HashMap::new(); + let mut client_server_states: HashMap = HashMap::new(); let mut now = Instant::now(); @@ -166,7 +227,7 @@ impl Collector { let stat = match self.rx.recv().await { Some(stat) => stat, None => { - println!(">> Statistics collector is shutting down."); + info!("Statistics collector is shutting down"); return; } }; @@ -212,20 +273,25 @@ impl Collector { } } - StatisticName::ClientActive | StatisticName::ClientWaiting | StatisticName::ClientIdle => { - client_states.insert(stat.process_id.unwrap(), stat.name); + StatisticName::ClientActive + | StatisticName::ClientWaiting + | StatisticName::ClientIdle + | StatisticName::ServerActive + | StatisticName::ServerIdle + | StatisticName::ServerTested + | StatisticName::ServerLogin => { + client_server_states.insert(stat.process_id.unwrap(), stat.name); } - StatisticName::ClientDisconnecting => { - client_states.remove(&stat.process_id.unwrap()); + StatisticName::ClientDisconnecting | StatisticName::ServerDisconnecting => { + client_server_states.remove(&stat.process_id.unwrap()); } }; - // It's been 15 seconds. If there is no traffic, it won't publish anything, // but it also doesn't matter then. if now.elapsed().as_secs() > 15 { - for (_, state) in &client_states { + for (_, state) in &client_server_states { match state { StatisticName::ClientActive => { let counter = stats.entry("cl_active").or_insert(0); @@ -242,11 +308,31 @@ impl Collector { *counter += 1; } + StatisticName::ServerIdle => { + let counter = stats.entry("sv_idle").or_insert(0); + *counter += 1; + } + + StatisticName::ServerActive => { + let counter = stats.entry("sv_active").or_insert(0); + *counter += 1; + } + + StatisticName::ServerTested => { + let counter = stats.entry("sv_tested").or_insert(0); + *counter += 1; + } + + StatisticName::ServerLogin => { + let counter = stats.entry("sv_login").or_insert(0); + *counter += 1; + } + _ => unreachable!(), }; } - println!(">> Reporting to StatsD: {:?}", stats); + info!("{:?}", stats); let mut pipeline = self.client.pipeline(); From c5a79dbce714afb42296bdf539dfea93c2f9b887 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 20 Feb 2022 22:49:30 -0800 Subject: [PATCH 097/551] Update CONTRIBUTING.md --- CONTRIBUTING.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 14714008f..b1d8b379b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,19 +3,17 @@ Thank you for contributing! Just a few tips here: 1. `cargo fmt` your code before opening up a PR -2. Run the "test suite" (i.e. PgBench) to make sure everything still works. +2. Run the test suite (e.g. `pgbench`) to make sure everything still works. The tests are in `.circleci/run_tests.sh`. +3. Performance is important, make sure there are no regressions in your branch vs. `main`. Happy hacking! ## TODOs -A non-exhaustive list of things that would be useful to implement. +A non-exhaustive list of things that would be useful to implement: #### Client authentication MD5 is probably sufficient, but maybe others too. -#### Statistics -Same as PgBouncer, e.g. client wait, transactions, timings, etc. I'm thinking we can use `mpsc` here ([docs](https://tokio.rs/tokio/tutorial/channels)), with clients sending stats and a task collecting and aggregating them. This should avoid atomics/mutexes. Caveat is the task should make sure not to crash, so the channels don't get backed up. - #### Admin Admin database for stats collection and pooler administration. PgBouncer gives us a nice example on how to do that, specifically how to implement `RowDescription` and `DataRow` messages, [example here](https://github.com/pgbouncer/pgbouncer/blob/4f9ced8e63d317a6ff45c8b0efa876b32161f6db/src/admin.c#L813). From ee5c0ef90ffde57f7dcc5407f5d9ef1320232d31 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 20 Feb 2022 23:33:04 -0800 Subject: [PATCH 098/551] Ruby (#30) * cop * log --- .circleci/config.yml | 2 + tests/ruby/tests.rb | 95 ++++++++++++++++---------------------------- 2 files changed, 37 insertions(+), 60 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f6180ad5a..6b330fee9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,8 @@ jobs: # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - image: cimg/rust:1.58.1 + environment: + RUST_LOG: info - image: cimg/postgres:14.0 auth: username: mydockerhub-user diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index 3ff39ab16..622ac0146 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -1,59 +1,27 @@ -require "active_record" +# frozen_string_literal: true + +require 'active_record' # Uncomment these two to see all queries. # ActiveRecord.verbose_query_logs = true # ActiveRecord::Base.logger = Logger.new(STDOUT) ActiveRecord::Base.establish_connection( - adapter: "postgresql", - host: "127.0.0.1", + adapter: 'postgresql', + host: '127.0.0.1', port: 6432, - username: "sharding_user", - password: "sharding_user", - database: "rails_dev", + username: 'sharding_user', + password: 'sharding_user', + database: 'rails_dev', prepared_statements: false, # Transaction mode - advisory_locks: false, # Same + advisory_locks: false # Same ) -class TestTable < ActiveRecord::Base - self.table_name = "test_table" -end - class TestSafeTable < ActiveRecord::Base - self.table_name = "test_safe_table" + self.table_name = 'test_safe_table' end -class ShouldNeverHappenException < Exception -end - -# # Create the table. -class CreateTestTable < ActiveRecord::Migration[7.0] - # Disable transasctions or things will fly out of order! - disable_ddl_transaction! - - SHARDS = 3 - - def change - SHARDS.times do |x| - # This will make this migration reversible! - reversible do - connection.execute "SET SHARD TO '#{x.to_i}'" - connection.execute "SET SERVER ROLE TO 'primary'" - end - - # Always wrap the entire migration inside a transaction. If that's not possible, - # execute a `SET SHARD` command before every statement and make sure AR doesn't need - # to load database information beforehand (i.e. it's not the first query in the migration). - connection.transaction do - create_table :test_table, if_not_exists: true do |t| - t.string :name - t.string :description - - t.timestamps - end - end - end - end +class ShouldNeverHappenException < RuntimeError end class CreateSafeShardedTable < ActiveRecord::Migration[7.0] @@ -85,44 +53,51 @@ def down SHARDS.times do |x| connection.execute "SET SHARD TO '#{x.to_i}'" connection.execute "SET SERVER ROLE TO 'primary'" - connection.execute "DROP TABLE test_safe_table CASCADE" + connection.execute 'DROP TABLE test_safe_table CASCADE' end end end -20.times do - begin - CreateTestTable.migrate(:down) - rescue Exception - puts "Tables don't exist yet" - end +SHARDS = 3 +2.times do begin CreateSafeShardedTable.migrate(:down) rescue Exception puts "Tables don't exist yet" end - CreateTestTable.migrate(:up) CreateSafeShardedTable.migrate(:up) - 3.times do |x| + SHARDS.times do |x| TestSafeTable.connection.execute "SET SHARD TO '#{x.to_i}'" TestSafeTable.connection.execute "SET SERVER ROLE TO 'primary'" - TestSafeTable.connection.execute "TRUNCATE #{TestTable.table_name}" + TestSafeTable.connection.execute "TRUNCATE #{TestSafeTable.table_name}" end - 10.times do |x| + # Equivalent to Makara's stick_to_master! except it sticks until it's changed. + TestSafeTable.connection.execute "SET SERVER ROLE TO 'primary'" + + 200.times do |x| x += 1 # Postgres ids start at 1 TestSafeTable.connection.execute "SET SHARDING KEY TO '#{x.to_i}'" - TestSafeTable.connection.execute "SET SERVER ROLE TO 'primary'" TestSafeTable.create(id: x, name: "something_special_#{x.to_i}", description: "It's a surprise!") end - 10.times do |x| + TestSafeTable.connection.execute "SET SERVER ROLE TO 'replica'" + + 100.times do |x| x += 1 # 0 confuses our sharding function TestSafeTable.connection.execute "SET SHARDING KEY TO '#{x.to_i}'" - TestSafeTable.connection.execute "SET SERVER ROLE TO 'replica'" + TestSafeTable.find_by_id(x).id + end + + # Will use the query parser to direct reads to replicas + TestSafeTable.connection.execute "SET SERVER ROLE TO 'auto'" + + 100.times do |x| + x += 101 + TestSafeTable.connection.execute "SET SHARDING KEY TO '#{x.to_i}'" TestSafeTable.find_by_id(x).id end end @@ -130,8 +105,8 @@ def down # Test wrong shard TestSafeTable.connection.execute "SET SHARD TO '1'" begin - TestSafeTable.create(id: 5, name: "test", description: "test description") - raise ShouldNeverHappenException("Uh oh") + TestSafeTable.create(id: 5, name: 'test', description: 'test description') + raise ShouldNeverHappenException('Uh oh') rescue ActiveRecord::StatementInvalid - puts "OK" + puts 'OK' end From 7ebbffb8f2e1f72e08ab5873003198d32cda8aca Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 20 Feb 2022 23:52:50 -0800 Subject: [PATCH 099/551] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 38 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..dd84ea782 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Smartphone (please complete the following information):** + - Device: [e.g. iPhone6] + - OS: [e.g. iOS8.1] + - Browser [e.g. stock browser, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..bbcbbe7d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From b49ed30914018b0c88821941714c7c9b3d6bd57e Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 21 Feb 2022 00:00:50 -0800 Subject: [PATCH 100/551] Reset query router setting to default (#32) --- .circleci/config.yml | 3 +++ src/pool.rs | 5 +---- src/query_router.rs | 14 ++++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b330fee9..63bfee52a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,9 @@ jobs: - checkout - restore_cache: key: cargo-lock-2-{{ checksum "Cargo.lock" }} + - run: + name: "Lint" + command: "cargo fmt --check" - run: name: "Install dependencies" command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev" diff --git a/src/pool.rs b/src/pool.rs index 87f169a3e..6c9a7a21a 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -167,10 +167,7 @@ impl ConnectionPool { }; if !exists { - error!( - "Requested role {:?}, but none are configured", - role - ); + error!("Requested role {:?}, but none are configured", role); return Err(Error::BadConfig); } diff --git a/src/query_router.rs b/src/query_router.rs index 64d6e7042..4134af8d4 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -3,12 +3,12 @@ use crate::sharding::Sharder; /// Route queries automatically based on explicitely requested /// or implied query characteristics. use bytes::{Buf, BytesMut}; +use log::{debug, error}; use once_cell::sync::OnceCell; use regex::RegexSet; use sqlparser::ast::Statement::{Query, StartTransaction}; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; -use log::{error, debug}; const CUSTOM_SQL_REGEXES: [&str; 5] = [ r"(?i)SET SHARDING KEY TO '[0-9]+'", @@ -174,6 +174,7 @@ impl QueryRouter { "default" => { // TODO: reset query parser to default here. self.active_role = self.default_server_role; + self.query_parser_enabled = get_config().query_router.query_parser_enabled; self.active_role } @@ -220,11 +221,7 @@ impl QueryRouter { let ast = match Parser::parse_sql(&PostgreSqlDialect {}, &query) { Ok(ast) => ast, Err(err) => { - debug!( - "{:?}, query: {}", - err, - query - ); + debug!("{:?}, query: {}", err, query); return false; } }; @@ -483,5 +480,10 @@ mod test { let query = simple_query("SELECT * FROM test_table"); assert_eq!(qr.infer_role(query), true); assert_eq!(qr.role(), Some(Role::Replica)); + + assert!(qr.query_parser_enabled()); + let query = simple_query("SET SERVER ROLE TO 'default'"); + assert!(qr.try_execute_command(query) != None); + assert!(!qr.query_parser_enabled()); } } From 1817452e705074add8914adf89e4d97b353d4e49 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 21 Feb 2022 00:16:33 -0800 Subject: [PATCH 101/551] test session mode and config reload for real (#33) * test session mode and config reload for real * period * run them at the end * typo * wrong dir --- .circleci/run_tests.sh | 13 ++++++++++--- README.md | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 9ba0686eb..2204b241f 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -34,9 +34,6 @@ psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql > / # Replica/primary selection & more sharding tests psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null -# Test reload config -kill -SIGHUP $(pgrep pgcat) - # # ActiveRecord tests! # @@ -45,6 +42,16 @@ sudo gem install bundler bundle install ruby tests.rb +cd ../../ +# Test session mode (and config reload) +sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' pgcat.toml + +# Reload config +kill -SIGHUP $(pgrep pgcat) + +# Prepared statements that will only work in session mode +pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol prepared + # Attempt clean shut down killall pgcat -s SIGINT diff --git a/README.md b/README.md index cf955eb25..38014732c 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ See [sharding README](./tests/sharding/README.md) for sharding logic testing. | **Feature** | **Tested in CI** | **Tested manually** | **Comments** | |-----------------------|--------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------| | Transaction pooling | :heavy_check_mark: | :heavy_check_mark: | Used by default for all tests. | -| Session pooling | :x: | :heavy_check_mark: | Easiest way to test is to enable it and run pgbench - results will be better than transaction pooling as expected. | +| Session pooling | :heavy_check_mark: | :heavy_check_mark: | Tested by running pgbench with `--protocol prepared` which only works in session mode. | | `COPY` | :heavy_check_mark: | :heavy_check_mark: | `pgbench -i` uses `COPY`. `COPY FROM` is tested as well. | | Query cancellation | :heavy_check_mark: | :heavy_check_mark: | `psql -c 'SELECT pg_sleep(1000);'` and press `Ctrl-C`. | | Load balancing | :x: | :heavy_check_mark: | We could test this by emitting statistics for each replica and compare them. | From fe29a6b1dc20da6ff3897f87fdebd11442aa4f56 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 21 Feb 2022 17:28:50 -0800 Subject: [PATCH 102/551] Add server login stat; refactor for better naming (#34) --- src/pool.rs | 19 ++++++- src/stats.rs | 156 +++++++++++++++++++++++++-------------------------- 2 files changed, 95 insertions(+), 80 deletions(-) diff --git a/src/pool.rs b/src/pool.rs index 6c9a7a21a..fb33fd851 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -351,7 +351,12 @@ impl ManageConnection for ServerPool { self.address, self.user.name ); - Server::startup( + // Put a temporary process_id into the stats + // for server login. + let process_id = rand::random::(); + self.stats.server_login(process_id); + + match Server::startup( &self.address, &self.user, &self.database, @@ -359,6 +364,18 @@ impl ManageConnection for ServerPool { self.stats.clone(), ) .await + { + Ok(conn) => { + // Remove the temporary process_id from the stats. + self.stats.server_disconnecting(process_id); + Ok(conn) + } + Err(err) => { + // Remove the temporary process_id from the stats. + self.stats.server_disconnecting(process_id); + Err(err) + } + } } /// Determines if the connection is still connected to the database. diff --git a/src/stats.rs b/src/stats.rs index beb430f11..0332b5f44 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,6 +1,6 @@ use log::info; use statsd::Client; -/// Statistics collector and publisher. +/// Events collector and publisher. use tokio::sync::mpsc::{Receiver, Sender}; use std::collections::HashMap; @@ -9,12 +9,10 @@ use std::time::Instant; use crate::config::get_config; #[derive(Debug, Clone, Copy)] -pub enum StatisticName { +enum EventName { CheckoutTime, - //QueryRuntime, - //TransactionTime, - Queries, - Transactions, + Query, + Transaction, DataSent, DataReceived, ClientWaiting, @@ -29,25 +27,25 @@ pub enum StatisticName { } #[derive(Debug)] -pub struct Statistic { - pub name: StatisticName, - pub value: i64, - pub process_id: Option, +pub struct Event { + name: EventName, + value: i64, + process_id: Option, } #[derive(Clone, Debug)] pub struct Reporter { - tx: Sender, + tx: Sender, } impl Reporter { - pub fn new(tx: Sender) -> Reporter { + pub fn new(tx: Sender) -> Reporter { Reporter { tx: tx } } - pub fn query(&mut self) { - let statistic = Statistic { - name: StatisticName::Queries, + pub fn query(&self) { + let statistic = Event { + name: EventName::Query, value: 1, process_id: None, }; @@ -55,9 +53,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn transaction(&mut self) { - let statistic = Statistic { - name: StatisticName::Transactions, + pub fn transaction(&self) { + let statistic = Event { + name: EventName::Transaction, value: 1, process_id: None, }; @@ -65,9 +63,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn data_sent(&mut self, amount: usize) { - let statistic = Statistic { - name: StatisticName::DataSent, + pub fn data_sent(&self, amount: usize) { + let statistic = Event { + name: EventName::DataSent, value: amount as i64, process_id: None, }; @@ -75,9 +73,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn data_received(&mut self, amount: usize) { - let statistic = Statistic { - name: StatisticName::DataReceived, + pub fn data_received(&self, amount: usize) { + let statistic = Event { + name: EventName::DataReceived, value: amount as i64, process_id: None, }; @@ -85,9 +83,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn checkout_time(&mut self, ms: u128) { - let statistic = Statistic { - name: StatisticName::CheckoutTime, + pub fn checkout_time(&self, ms: u128) { + let statistic = Event { + name: EventName::CheckoutTime, value: ms as i64, process_id: None, }; @@ -95,9 +93,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn client_waiting(&mut self, process_id: i32) { - let statistic = Statistic { - name: StatisticName::ClientWaiting, + pub fn client_waiting(&self, process_id: i32) { + let statistic = Event { + name: EventName::ClientWaiting, value: 1, process_id: Some(process_id), }; @@ -105,9 +103,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn client_active(&mut self, process_id: i32) { - let statistic = Statistic { - name: StatisticName::ClientActive, + pub fn client_active(&self, process_id: i32) { + let statistic = Event { + name: EventName::ClientActive, value: 1, process_id: Some(process_id), }; @@ -115,9 +113,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn client_idle(&mut self, process_id: i32) { - let statistic = Statistic { - name: StatisticName::ClientIdle, + pub fn client_idle(&self, process_id: i32) { + let statistic = Event { + name: EventName::ClientIdle, value: 1, process_id: Some(process_id), }; @@ -125,9 +123,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn client_disconnecting(&mut self, process_id: i32) { - let statistic = Statistic { - name: StatisticName::ClientDisconnecting, + pub fn client_disconnecting(&self, process_id: i32) { + let statistic = Event { + name: EventName::ClientDisconnecting, value: 1, process_id: Some(process_id), }; @@ -135,9 +133,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn server_active(&mut self, process_id: i32) { - let statistic = Statistic { - name: StatisticName::ServerActive, + pub fn server_active(&self, process_id: i32) { + let statistic = Event { + name: EventName::ServerActive, value: 1, process_id: Some(process_id), }; @@ -145,9 +143,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn server_idle(&mut self, process_id: i32) { - let statistic = Statistic { - name: StatisticName::ServerIdle, + pub fn server_idle(&self, process_id: i32) { + let statistic = Event { + name: EventName::ServerIdle, value: 1, process_id: Some(process_id), }; @@ -155,9 +153,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn server_login(&mut self, process_id: i32) { - let statistic = Statistic { - name: StatisticName::ServerLogin, + pub fn server_login(&self, process_id: i32) { + let statistic = Event { + name: EventName::ServerLogin, value: 1, process_id: Some(process_id), }; @@ -165,9 +163,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn server_tested(&mut self, process_id: i32) { - let statistic = Statistic { - name: StatisticName::ServerTested, + pub fn server_tested(&self, process_id: i32) { + let statistic = Event { + name: EventName::ServerTested, value: 1, process_id: Some(process_id), }; @@ -175,9 +173,9 @@ impl Reporter { let _ = self.tx.try_send(statistic); } - pub fn server_disconnecting(&mut self, process_id: i32) { - let statistic = Statistic { - name: StatisticName::ServerDisconnecting, + pub fn server_disconnecting(&self, process_id: i32) { + let statistic = Event { + name: EventName::ServerDisconnecting, value: 1, process_id: Some(process_id), }; @@ -187,12 +185,12 @@ impl Reporter { } pub struct Collector { - rx: Receiver, + rx: Receiver, client: Client, } impl Collector { - pub fn new(rx: Receiver) -> Collector { + pub fn new(rx: Receiver) -> Collector { Collector { rx: rx, client: Client::new(&get_config().general.statsd_address, "pgcat").unwrap(), @@ -200,7 +198,7 @@ impl Collector { } pub async fn collect(&mut self) { - info!("Statistics reporter started"); + info!("Events reporter started"); let mut stats = HashMap::from([ ("total_query_count", 0), @@ -219,7 +217,7 @@ impl Collector { ("sv_tested", 0), ]); - let mut client_server_states: HashMap = HashMap::new(); + let mut client_server_states: HashMap = HashMap::new(); let mut now = Instant::now(); @@ -227,34 +225,34 @@ impl Collector { let stat = match self.rx.recv().await { Some(stat) => stat, None => { - info!("Statistics collector is shutting down"); + info!("Events collector is shutting down"); return; } }; // Some are counters, some are gauges... match stat.name { - StatisticName::Queries => { + EventName::Query => { let counter = stats.entry("total_query_count").or_insert(0); *counter += stat.value; } - StatisticName::Transactions => { + EventName::Transaction => { let counter = stats.entry("total_xact_count").or_insert(0); *counter += stat.value; } - StatisticName::DataSent => { + EventName::DataSent => { let counter = stats.entry("total_sent").or_insert(0); *counter += stat.value; } - StatisticName::DataReceived => { + EventName::DataReceived => { let counter = stats.entry("total_received").or_insert(0); *counter += stat.value; } - StatisticName::CheckoutTime => { + EventName::CheckoutTime => { let counter = stats.entry("total_wait_time").or_insert(0); *counter += stat.value; @@ -273,17 +271,17 @@ impl Collector { } } - StatisticName::ClientActive - | StatisticName::ClientWaiting - | StatisticName::ClientIdle - | StatisticName::ServerActive - | StatisticName::ServerIdle - | StatisticName::ServerTested - | StatisticName::ServerLogin => { + EventName::ClientActive + | EventName::ClientWaiting + | EventName::ClientIdle + | EventName::ServerActive + | EventName::ServerIdle + | EventName::ServerTested + | EventName::ServerLogin => { client_server_states.insert(stat.process_id.unwrap(), stat.name); } - StatisticName::ClientDisconnecting | StatisticName::ServerDisconnecting => { + EventName::ClientDisconnecting | EventName::ServerDisconnecting => { client_server_states.remove(&stat.process_id.unwrap()); } }; @@ -293,37 +291,37 @@ impl Collector { if now.elapsed().as_secs() > 15 { for (_, state) in &client_server_states { match state { - StatisticName::ClientActive => { + EventName::ClientActive => { let counter = stats.entry("cl_active").or_insert(0); *counter += 1; } - StatisticName::ClientWaiting => { + EventName::ClientWaiting => { let counter = stats.entry("cl_waiting").or_insert(0); *counter += 1; } - StatisticName::ClientIdle => { + EventName::ClientIdle => { let counter = stats.entry("cl_idle").or_insert(0); *counter += 1; } - StatisticName::ServerIdle => { + EventName::ServerIdle => { let counter = stats.entry("sv_idle").or_insert(0); *counter += 1; } - StatisticName::ServerActive => { + EventName::ServerActive => { let counter = stats.entry("sv_active").or_insert(0); *counter += 1; } - StatisticName::ServerTested => { + EventName::ServerTested => { let counter = stats.entry("sv_tested").or_insert(0); *counter += 1; } - StatisticName::ServerLogin => { + EventName::ServerLogin => { let counter = stats.entry("sv_login").or_insert(0); *counter += 1; } From 3f994380d9b6385da977d7d2f225aa59d9557e3f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 21 Feb 2022 17:48:08 -0800 Subject: [PATCH 103/551] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 38014732c..7e34cb3b8 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ That setting can be adjusted to spawn as many (or as little) workers as needed. 2. `cargo build --release` (to get better benchmarks). 3. Change the config in `pgcat.toml` to fit your setup (optional given next step). 4. Install Postgres and run `psql -f tests/sharding/query_routing_setup.sql` (user/password may be required depending on your setup) -5. `cargo run --release` You're ready to go! +5. `RUST_LOG=info cargo run --release` You're ready to go! ### Tests From 80b54707cc324827fe9b96192846a14ff1864cf4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 21 Feb 2022 17:49:32 -0800 Subject: [PATCH 104/551] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 37706259d..cdd0e557c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,5 @@ FROM debian:buster-slim COPY --from=builder /app/target/release/pgcat /usr/bin/pgcat COPY --from=builder /app/pgcat.toml /etc/pgcat/pgcat.toml WORKDIR /etc/pgcat +ENV RUST_LOG=info ENTRYPOINT ["/usr/bin/pgcat"] From b3cd7dae4def06eda7fe3834e6e0b5c499ef6e50 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 21 Feb 2022 20:41:32 -0800 Subject: [PATCH 105/551] Can pass config path as argument (#36) * show better errors for config parsing * lint --- src/config.rs | 17 ++++++++++++++--- src/main.rs | 10 +++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/config.rs b/src/config.rs index 221fb6673..04a81628f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -172,7 +172,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { let mut file = match File::open(path).await { Ok(file) => file, Err(err) => { - error!("{:?}", err); + error!("Could not open '{}': {}", path, err.to_string()); return Err(Error::BadConfig); } }; @@ -180,7 +180,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { match file.read_to_string(&mut contents).await { Ok(_) => (), Err(err) => { - error!("{:?}", err); + error!("Could not read config file: {}", err.to_string()); return Err(Error::BadConfig); } }; @@ -188,7 +188,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { let config: Config = match toml::from_str(&contents) { Ok(config) => config, Err(err) => { - error!("{:?}", err); + error!("Could not parse config file: {}", err.to_string()); return Err(Error::BadConfig); } }; @@ -200,6 +200,17 @@ pub async fn parse(path: &str) -> Result<(), Error> { let mut dup_check = HashSet::new(); let mut primary_count = 0; + match shard.0.parse::() { + Ok(_) => (), + Err(_) => { + error!( + "Shard '{}' is not a valid number, shards must be numbered starting at 0", + shard.0 + ); + return Err(Error::BadConfig); + } + }; + if shard.1.servers.len() == 0 { error!("Shard {} has no servers configured", shard.0); return Err(Error::BadConfig); diff --git a/src/main.rs b/src/main.rs index 65a12c9a3..72767c124 100644 --- a/src/main.rs +++ b/src/main.rs @@ -75,8 +75,16 @@ async fn main() { return; } + let args = std::env::args().collect::>(); + + let config_file = if args.len() == 2 { + args[1].to_string() + } else { + String::from("pgcat.toml") + }; + // Prepare the config - match config::parse("pgcat.toml").await { + match config::parse(&config_file).await { Ok(_) => (), Err(err) => { error!("Config parse error: {:?}", err); From adfb87854118947ad044c70878379ffd72c6710d Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 21 Feb 2022 20:43:40 -0800 Subject: [PATCH 106/551] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7e34cb3b8..37f6c77be 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover support. -**Alpha**: don't use in production just yet. +**Alpha**: looking for alpha testers, see [#35](https://github.com/levkk/pgcat/issues/35). ## Features | **Feature** | **Status** | **Comments** | From 2085d7412bce472349c09bf449e5f902b84d7602 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 21 Feb 2022 23:57:25 -0800 Subject: [PATCH 107/551] Config docs (#37) * config docs * space * space * shards --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 37f6c77be..b6617377e 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,32 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su See `Dockerfile` for example deployment using Docker. The pooler is configured to spawn 4 workers so 4 CPUs are recommended for optimal performance. That setting can be adjusted to spawn as many (or as little) workers as needed. +### Config + +| **Name** | **Description** | **Examples** | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------| +| **`general`** | | | +| `host` | The pooler will run on this host, 0.0.0.0 means accessible from everywhere. | `0.0.0.0` | +| `port` | The pooler will run on this port. | `6432` | +| `pool_size` | Maximum allowed server connections per pool. Pools are separated for each user/shard/server role. The connections are allocated as needed. | `15` | +| `pool_mode` | The pool mode to use, i.e. `session` or `transaction`. | `transaction` | +| `connect_timeout` | Maximum time to establish a connection to a server (milliseconds). If reached, the server is banned and the next target is attempted. | `5000` | +| `healthcheck_timeout` | Maximum time to pass a health check (`SELECT 1`, milliseconds). If reached, the server is banned and the next target is attempted. | `1000` | +| `ban_time` | Ban time for a server (seconds). It won't be allowed to serve transactions until the ban expires; failover targets will be used instead. | `60` | +| `statsd_address` | StatsD host and port. Statistics will be sent there every 15 seconds. | `127.0.0.1:8125` | +| | | | +| **`user`** | | | +| `name` | The user name. | `sharding_user` | +| `password` | The user password in plaintext. | `hunter2` | +| | | | +| **`shards`** | Shards are numerically numbered starting from 0; the order in the config is preserved by the pooler to route queries accordingly. | `[shards.0]` | +| `servers` | List of servers to connect to and their roles. A server is: `[host, port, role]`, where `role` is either `primary` or `replica`. | `["127.0.0.1", 5432, "primary"]` | +| `database` | The name of the database to connect to. This is the same on all servers that are part of one shard. | | +| **`query_router`** | | | +| `default_role` | Traffic is routed to this role by default (round-robin), unless the client specifies otherwise. Default is `any`, for any role available. | `any`, `primary`, `replica` | +| `query_parser_enabled` | Enable the query parser which will inspect incoming queries and route them to a primary or replicas. | `false` | +| `primary_reads_enabled` | Enable this to allow read queries on the primary; otherwise read queries are routed to the replicas. | `true` | + ## Local development 1. Install Rust (latest stable will work great). From e75e506210b152d0c3c07335dba896391ae7d173 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 22 Feb 2022 18:10:30 -0800 Subject: [PATCH 108/551] Flush stats (#38) * flush stats * stats * refactor --- README.md | 34 ++++----- src/main.rs | 4 +- src/stats.rs | 193 ++++++++++++++++++++++++++++----------------------- 3 files changed, 125 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index b6617377e..b05eaafd5 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,14 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su ## Features | **Feature** | **Status** | **Comments** | |--------------------------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| -| Transaction pooling | :heavy_check_mark: | Identical to PgBouncer. | -| Session pooling | :heavy_check_mark: | Identical to PgBouncer. | -| `COPY` support | :heavy_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | -| Query cancellation | :heavy_check_mark: | Supported both in transaction and session pooling modes. | -| Load balancing of read queries | :heavy_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | -| Sharding | :heavy_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | -| Failover | :heavy_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | -| Statistics reporting | :heavy_check_mark: | Statistics similar to PgBouncers are reported via StatsD. | +| Transaction pooling | :white_check_mark: | Identical to PgBouncer. | +| Session pooling | :white_check_mark: | Identical to PgBouncer. | +| `COPY` support | :white_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | +| Query cancellation | :white_check_mark: | Supported both in transaction and session pooling modes. | +| Load balancing of read queries | :white_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | +| Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | +| Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | +| Statistics reporting | :white_check_mark: | Statistics similar to PgBouncers are reported via StatsD. | | Live configuration reloading | :construction_worker: | Reload config with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)`. Not all settings can be reloaded without a restart. | | Client authentication | :x: :wrench: | On the roadmap; currently all clients are allowed to connect and one user is used to connect to Postgres. | @@ -75,15 +75,15 @@ See [sharding README](./tests/sharding/README.md) for sharding logic testing. | **Feature** | **Tested in CI** | **Tested manually** | **Comments** | |-----------------------|--------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------| -| Transaction pooling | :heavy_check_mark: | :heavy_check_mark: | Used by default for all tests. | -| Session pooling | :heavy_check_mark: | :heavy_check_mark: | Tested by running pgbench with `--protocol prepared` which only works in session mode. | -| `COPY` | :heavy_check_mark: | :heavy_check_mark: | `pgbench -i` uses `COPY`. `COPY FROM` is tested as well. | -| Query cancellation | :heavy_check_mark: | :heavy_check_mark: | `psql -c 'SELECT pg_sleep(1000);'` and press `Ctrl-C`. | -| Load balancing | :x: | :heavy_check_mark: | We could test this by emitting statistics for each replica and compare them. | -| Failover | :x: | :heavy_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing could include using Toxiproxy. | -| Sharding | :heavy_check_mark: | :heavy_check_mark: | See `tests/sharding` and `tests/ruby` for an Rails/ActiveRecord example. | -| Statistics reporting | :x: | :heavy_check_mark: | Run `nc -l -u 8125` and watch the stats come in every 15 seconds. | -| Live config reloading | :heavy_check_mark: | :heavy_check_mark: | Run `kill -s SIGHUP $(pgrep pgcat)` and watch the config reload. | +| Transaction pooling | :white_check_mark: | :white_check_mark: | Used by default for all tests. | +| Session pooling | :white_check_mark: | :white_check_mark: | Tested by running pgbench with `--protocol prepared` which only works in session mode. | +| `COPY` | :white_check_mark: | :white_check_mark: | `pgbench -i` uses `COPY`. `COPY FROM` is tested as well. | +| Query cancellation | :white_check_mark: | :white_check_mark: | `psql -c 'SELECT pg_sleep(1000);'` and press `Ctrl-C`. | +| Load balancing | :x: | :white_check_mark: | We could test this by emitting statistics for each replica and compare them. | +| Failover | :x: | :white_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing could include using Toxiproxy. | +| Sharding | :white_check_mark: | :white_check_mark: | See `tests/sharding` and `tests/ruby` for an Rails/ActiveRecord example. | +| Statistics reporting | :x: | :white_check_mark: | Run `nc -l -u 8125` and watch the stats come in every 15 seconds. | +| Live config reloading | :white_check_mark: | :white_check_mark: | Run `kill -s SIGHUP $(pgrep pgcat)` and watch the config reload. | ## Usage diff --git a/src/main.rs b/src/main.rs index 72767c124..21da9aff9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -111,9 +111,9 @@ async fn main() { // Collect statistics and send them to StatsD let (tx, rx) = mpsc::channel(100); - + let collector_tx = tx.clone(); tokio::task::spawn(async move { - let mut stats_collector = Collector::new(rx); + let mut stats_collector = Collector::new(rx, collector_tx); stats_collector.collect().await; }); diff --git a/src/stats.rs b/src/stats.rs index 0332b5f44..384aa1a26 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -4,7 +4,6 @@ use statsd::Client; use tokio::sync::mpsc::{Receiver, Sender}; use std::collections::HashMap; -use std::time::Instant; use crate::config::get_config; @@ -24,6 +23,7 @@ enum EventName { ServerTested, ServerLogin, ServerDisconnecting, + FlushStatsToStatsD, } #[derive(Debug)] @@ -44,155 +44,167 @@ impl Reporter { } pub fn query(&self) { - let statistic = Event { + let event = Event { name: EventName::Query, value: 1, process_id: None, }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn transaction(&self) { - let statistic = Event { + let event = Event { name: EventName::Transaction, value: 1, process_id: None, }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn data_sent(&self, amount: usize) { - let statistic = Event { + let event = Event { name: EventName::DataSent, value: amount as i64, process_id: None, }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn data_received(&self, amount: usize) { - let statistic = Event { + let event = Event { name: EventName::DataReceived, value: amount as i64, process_id: None, }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn checkout_time(&self, ms: u128) { - let statistic = Event { + let event = Event { name: EventName::CheckoutTime, value: ms as i64, process_id: None, }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn client_waiting(&self, process_id: i32) { - let statistic = Event { + let event = Event { name: EventName::ClientWaiting, value: 1, process_id: Some(process_id), }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn client_active(&self, process_id: i32) { - let statistic = Event { + let event = Event { name: EventName::ClientActive, value: 1, process_id: Some(process_id), }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn client_idle(&self, process_id: i32) { - let statistic = Event { + let event = Event { name: EventName::ClientIdle, value: 1, process_id: Some(process_id), }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn client_disconnecting(&self, process_id: i32) { - let statistic = Event { + let event = Event { name: EventName::ClientDisconnecting, value: 1, process_id: Some(process_id), }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn server_active(&self, process_id: i32) { - let statistic = Event { + let event = Event { name: EventName::ServerActive, value: 1, process_id: Some(process_id), }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn server_idle(&self, process_id: i32) { - let statistic = Event { + let event = Event { name: EventName::ServerIdle, value: 1, process_id: Some(process_id), }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn server_login(&self, process_id: i32) { - let statistic = Event { + let event = Event { name: EventName::ServerLogin, value: 1, process_id: Some(process_id), }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn server_tested(&self, process_id: i32) { - let statistic = Event { + let event = Event { name: EventName::ServerTested, value: 1, process_id: Some(process_id), }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } pub fn server_disconnecting(&self, process_id: i32) { - let statistic = Event { + let event = Event { name: EventName::ServerDisconnecting, value: 1, process_id: Some(process_id), }; - let _ = self.tx.try_send(statistic); + let _ = self.tx.try_send(event); } + + // pub fn flush_to_statsd(&self) { + // let event = Event { + // name: EventName::FlushStatsToStatsD, + // value: 0, + // process_id: None, + // }; + + // let _ = self.tx.try_send(event); + // } } pub struct Collector { rx: Receiver, + tx: Sender, client: Client, } impl Collector { - pub fn new(rx: Receiver) -> Collector { + pub fn new(rx: Receiver, tx: Sender) -> Collector { Collector { - rx: rx, + rx, + tx, client: Client::new(&get_config().general.statsd_address, "pgcat").unwrap(), } } @@ -218,8 +230,19 @@ impl Collector { ]); let mut client_server_states: HashMap = HashMap::new(); - - let mut now = Instant::now(); + let tx = self.tx.clone(); + + tokio::task::spawn(async move { + let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(15000)); + loop { + interval.tick().await; + let _ = tx.try_send(Event { + name: EventName::FlushStatsToStatsD, + value: 0, + process_id: None, + }); + } + }); loop { let stat = match self.rx.recv().await { @@ -284,65 +307,61 @@ impl Collector { EventName::ClientDisconnecting | EventName::ServerDisconnecting => { client_server_states.remove(&stat.process_id.unwrap()); } - }; - - // It's been 15 seconds. If there is no traffic, it won't publish anything, - // but it also doesn't matter then. - if now.elapsed().as_secs() > 15 { - for (_, state) in &client_server_states { - match state { - EventName::ClientActive => { - let counter = stats.entry("cl_active").or_insert(0); - *counter += 1; - } - - EventName::ClientWaiting => { - let counter = stats.entry("cl_waiting").or_insert(0); - *counter += 1; - } - - EventName::ClientIdle => { - let counter = stats.entry("cl_idle").or_insert(0); - *counter += 1; - } - - EventName::ServerIdle => { - let counter = stats.entry("sv_idle").or_insert(0); - *counter += 1; - } - - EventName::ServerActive => { - let counter = stats.entry("sv_active").or_insert(0); - *counter += 1; - } - - EventName::ServerTested => { - let counter = stats.entry("sv_tested").or_insert(0); - *counter += 1; - } - - EventName::ServerLogin => { - let counter = stats.entry("sv_login").or_insert(0); - *counter += 1; - } - - _ => unreachable!(), - }; - } - info!("{:?}", stats); + EventName::FlushStatsToStatsD => { + for (_, state) in &client_server_states { + match state { + EventName::ClientActive => { + let counter = stats.entry("cl_active").or_insert(0); + *counter += 1; + } + + EventName::ClientWaiting => { + let counter = stats.entry("cl_waiting").or_insert(0); + *counter += 1; + } + + EventName::ClientIdle => { + let counter = stats.entry("cl_idle").or_insert(0); + *counter += 1; + } + + EventName::ServerIdle => { + let counter = stats.entry("sv_idle").or_insert(0); + *counter += 1; + } + + EventName::ServerActive => { + let counter = stats.entry("sv_active").or_insert(0); + *counter += 1; + } + + EventName::ServerTested => { + let counter = stats.entry("sv_tested").or_insert(0); + *counter += 1; + } + + EventName::ServerLogin => { + let counter = stats.entry("sv_login").or_insert(0); + *counter += 1; + } + + _ => unreachable!(), + }; + } - let mut pipeline = self.client.pipeline(); + info!("{:?}", stats); - for (key, value) in stats.iter_mut() { - pipeline.gauge(key, *value as f64); - *value = 0; - } + let mut pipeline = self.client.pipeline(); - pipeline.send(&self.client); + for (key, value) in stats.iter_mut() { + pipeline.gauge(key, *value as f64); + *value = 0; + } - now = Instant::now(); - } + pipeline.send(&self.client); + } + }; } } } From 9517fa9c4eb6bc2a201aef277e3e98285454d65e Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 22 Feb 2022 19:26:08 -0800 Subject: [PATCH 109/551] Add debug logging (#39) * Add debug for easier debugging * fmt * a couple more messages --- src/client.rs | 41 +++++++++++++++++++++++++++++++++++++++-- src/query_router.rs | 11 +++++++++-- src/server.rs | 14 +++++++++++++- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/client.rs b/src/client.rs index 06f5a9f65..096125305 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,7 +2,7 @@ /// We are pretending to the server in this scenario, /// and this module implements that. use bytes::{Buf, BufMut, BytesMut}; -use log::error; +use log::{debug, error}; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, @@ -70,6 +70,8 @@ impl Client { let transaction_mode = config.general.pool_mode.starts_with("t"); drop(config); loop { + debug!("Waiting for StartupMessage"); + // Could be StartupMessage or SSLRequest // which makes this variable length. let len = match stream.read_i32().await { @@ -91,6 +93,8 @@ impl Client { match code { // Client wants SSL. We don't support it at the moment. SSL_REQUEST_CODE => { + debug!("Rejecting SSLRequest"); + let mut no = BytesMut::with_capacity(1); no.put_u8(b'N'); @@ -99,6 +103,8 @@ impl Client { // Regular startup message. PROTOCOL_VERSION_NUMBER => { + debug!("Got StartupMessage"); + // TODO: perform actual auth. let parameters = parse_startup(bytes.clone())?; @@ -110,6 +116,7 @@ impl Client { write_all(&mut stream, server_info).await?; backend_key_data(&mut stream, process_id, secret_key).await?; ready_for_query(&mut stream).await?; + debug!("Startup OK"); // Split the read and write streams // so we can control buffering. @@ -161,6 +168,8 @@ impl Client { pub async fn handle(&mut self, mut pool: ConnectionPool) -> Result<(), Error> { // The client wants to cancel a query it has issued previously. if self.cancel_mode { + debug!("Sending CancelRequest"); + let (process_id, secret_key, address, port) = { let guard = self.client_server_map.lock().unwrap(); @@ -193,6 +202,8 @@ impl Client { // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocols. loop { + debug!("Client idle, waiting for message"); + // Client idle, waiting for messages. self.stats.client_idle(self.process_id); @@ -203,6 +214,12 @@ impl Client { // SET SHARDING KEY TO 'bigint'; let mut message = read_message(&mut self.read).await?; + // Avoid taking a server if the client just wants to disconnect. + if message[0] as char == 'X' { + debug!("Client disconnecting"); + return Ok(()); + } + // Handle all custom protocol commands here. match query_router.try_execute_command(message.clone()) { // Normal query @@ -250,9 +267,14 @@ impl Client { // Waiting for server connection. self.stats.client_waiting(self.process_id); + debug!("Waiting for connection from pool"); + // Grab a server from the pool: the client issued a regular query. let connection = match pool.get(query_router.shard(), query_router.role()).await { - Ok(conn) => conn, + Ok(conn) => { + debug!("Got connection from pool"); + conn + } Err(err) => { error!("Could not get connection from pool: {:?}", err); error_response(&mut self.write, "could not get connection from the pool") @@ -272,11 +294,19 @@ impl Client { self.stats.client_active(self.process_id); self.stats.server_active(server.process_id()); + debug!( + "Client {:?} talking to server {:?}", + self.write.peer_addr().unwrap(), + server.address() + ); + // Transaction loop. Multiple queries can be issued by the client here. // The connection belongs to the client until the transaction is over, // or until the client disconnects if we are in session mode. loop { let mut message = if message.len() == 0 { + debug!("Waiting for message inside transaction or in session mode"); + match read_message(&mut self.read).await { Ok(message) => message, Err(err) => { @@ -303,9 +333,13 @@ impl Client { let code = message.get_u8() as char; let _len = message.get_i32() as usize; + debug!("Message: {}", code); + match code { // ReadyForQuery 'Q' => { + debug!("Sending query to server"); + // TODO: implement retries here for read-only transactions. server.send(original).await?; @@ -387,6 +421,8 @@ impl Client { // Sync // Frontend (client) is asking for the query result now. 'S' => { + debug!("Sending query to server"); + self.buffer.put(&original[..]); // TODO: retries for read-only transactions. @@ -471,6 +507,7 @@ impl Client { } // The server is no longer bound to us, we can't cancel it's queries anymore. + debug!("Releasing server back into the pool"); self.release(); } } diff --git a/src/query_router.rs b/src/query_router.rs index 4134af8d4..8dddef077 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -194,7 +194,12 @@ impl QueryRouter { let len = buf.get_i32() as usize; let query = match code { - 'Q' => String::from_utf8_lossy(&buf[..len - 5]).to_string(), + 'Q' => { + let query = String::from_utf8_lossy(&buf[..len - 5]).to_string(); + debug!("Query: '{}'", query); + query + } + 'P' => { let mut start = 0; let mut end; @@ -213,6 +218,8 @@ impl QueryRouter { let query = String::from_utf8_lossy(&buf[start..end]).to_string(); + debug!("Prepared statement: '{}'", query); + query.replace("$", "") // Remove placeholders turning them into "values" } _ => return false, @@ -221,7 +228,7 @@ impl QueryRouter { let ast = match Parser::parse_sql(&PostgreSqlDialect {}, &query) { Ok(ast) => ast, Err(err) => { - debug!("{:?}, query: {}", err, query); + debug!("{}", err.to_string()); return false; } }; diff --git a/src/server.rs b/src/server.rs index 18b1f267e..f186c0c75 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,7 +1,7 @@ use bytes::{Buf, BufMut, BytesMut}; ///! Implementation of the PostgreSQL server (database) protocol. ///! Here we are pretending to the a Postgres client. -use log::{error, info}; +use log::{debug, error, info}; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, @@ -75,6 +75,8 @@ impl Server { } }; + debug!("Sending StartupMessage"); + // Send the startup packet telling the server we're a normal Postgres client. startup(&mut stream, &user.name, database).await?; @@ -95,6 +97,8 @@ impl Server { Err(_) => return Err(Error::SocketError), }; + debug!("Message: {}", code); + match code { // Authentication 'R' => { @@ -104,6 +108,8 @@ impl Server { Err(_) => return Err(Error::SocketError), }; + debug!("Auth: {}", auth_code); + match auth_code { MD5_ENCRYPTED_PASSWORD => { // The salt is 4 bytes. @@ -135,6 +141,8 @@ impl Server { Err(_) => return Err(Error::SocketError), }; + debug!("Error: {}", error_code); + match error_code { // No error message is present in the message. MESSAGE_TERMINATOR => (), @@ -247,6 +255,8 @@ impl Server { } }; + debug!("Sending CancelRequest"); + let mut bytes = BytesMut::with_capacity(16); bytes.put_i32(16); bytes.put_i32(CANCEL_REQUEST_CODE); @@ -290,6 +300,8 @@ impl Server { let code = message.get_u8() as char; let _len = message.get_i32(); + debug!("Message: {}", code); + match code { // ReadyForQuery 'Z' => { From bd226181768bddc871be10aa52bdb2e743942de2 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 23 Feb 2022 11:47:24 -0800 Subject: [PATCH 110/551] Another example of a sharding function (#41) * Another example of a sharding function * tests --- pgcat.toml | 10 +++++++ src/config.rs | 16 +++++++++++ src/query_router.rs | 16 +++++++++-- src/sharding.rs | 69 +++++++++++++++++++++++++++++++++++++++------ 4 files changed, 99 insertions(+), 12 deletions(-) diff --git a/pgcat.toml b/pgcat.toml index 9a240b3c9..213180237 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -96,3 +96,13 @@ query_parser_enabled = false # load balancing of read queries. Otherwise, the primary will only be used for write # queries. The primary can always be explicitely selected with our custom protocol. primary_reads_enabled = true + +# So what if you wanted to implement a different hashing function, +# or you've already built one and you want this pooler to use it? +# +# Current options: +# +# pg_bigint_hash: PARTITION BY HASH (Postgres hashing function) +# sha1: A hashing function based on SHA1 +# +sharding_function = "pg_bigint_hash" diff --git a/src/config.rs b/src/config.rs index 04a81628f..754dc4fb2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -118,6 +118,7 @@ pub struct QueryRouter { pub default_role: String, pub query_parser_enabled: bool, pub primary_reads_enabled: bool, + pub sharding_function: String, } impl Default for QueryRouter { @@ -126,6 +127,7 @@ impl Default for QueryRouter { default_role: String::from("any"), query_parser_enabled: false, primary_reads_enabled: true, + sharding_function: "pg_bigint_hash".to_string(), } } } @@ -159,6 +161,8 @@ impl Config { self.general.healthcheck_timeout ); info!("Connection timeout: {}ms", self.general.connect_timeout); + info!("Sharding function: {}", self.query_router.sharding_function); + info!("Number of shards: {}", self.shards.len()); } } @@ -193,6 +197,18 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; + match config.query_router.sharding_function.as_ref() { + "pg_bigint_hash" => (), + "sha1" => (), + _ => { + error!( + "Supported sharding functions are: 'pg_bigint_hash', 'sha1', got: '{}'", + config.query_router.sharding_function + ); + return Err(Error::BadConfig); + } + }; + // Quick config sanity check. for shard in &config.shards { // We use addresses as unique identifiers, diff --git a/src/query_router.rs b/src/query_router.rs index 8dddef077..3f309ea7b 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1,5 +1,5 @@ use crate::config::{get_config, Role}; -use crate::sharding::Sharder; +use crate::sharding::{Sharder, ShardingFunction}; /// Route queries automatically based on explicitely requested /// or implied query characteristics. use bytes::{Buf, BytesMut}; @@ -48,6 +48,9 @@ pub struct QueryRouter { // Should we try to parse queries? query_parser_enabled: bool, + + // Which sharding function are we using? + sharding_function: ShardingFunction, } impl QueryRouter { @@ -76,6 +79,12 @@ impl QueryRouter { _ => unreachable!(), }; + let sharding_function = match config.query_router.sharding_function.as_ref() { + "pg_bigint_hash" => ShardingFunction::PgBigintHash, + "sha1" => ShardingFunction::Sha1, + _ => unreachable!(), + }; + QueryRouter { default_server_role: default_server_role, shards: config.shards.len(), @@ -84,6 +93,7 @@ impl QueryRouter { active_shard: None, primary_reads_enabled: config.query_router.primary_reads_enabled, query_parser_enabled: config.query_router.query_parser_enabled, + sharding_function, } } @@ -139,8 +149,8 @@ impl QueryRouter { match command { Command::SetShardingKey => { - let sharder = Sharder::new(self.shards); - let shard = sharder.pg_bigint_hash(value.parse::().unwrap()); + let sharder = Sharder::new(self.shards, self.sharding_function); + let shard = sharder.shard(value.parse::().unwrap()); self.active_shard = Some(shard); value = shard.to_string(); } diff --git a/src/sharding.rs b/src/sharding.rs index 7aed4d306..23eccc142 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -1,26 +1,62 @@ +use sha1::{Digest, Sha1}; + // https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/include/catalog/partition.h#L20 const PARTITION_HASH_SEED: u64 = 0x7A5B22367996DCFD; +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum ShardingFunction { + PgBigintHash, + Sha1, +} + pub struct Sharder { shards: usize, + sharding_function: ShardingFunction, } impl Sharder { - pub fn new(shards: usize) -> Sharder { - Sharder { shards: shards } + pub fn new(shards: usize, sharding_function: ShardingFunction) -> Sharder { + Sharder { + shards, + sharding_function, + } + } + + pub fn shard(&self, key: i64) -> usize { + match self.sharding_function { + ShardingFunction::PgBigintHash => self.pg_bigint_hash(key), + ShardingFunction::Sha1 => self.sha1(key), + } } /// Hash function used by Postgres to determine which partition /// to put the row in when using HASH(column) partitioning. /// Source: https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/common/hashfn.c#L631 /// Supports only 1 bigint at the moment, but we can add more later. - pub fn pg_bigint_hash(&self, key: i64) -> usize { + fn pg_bigint_hash(&self, key: i64) -> usize { let mut lohalf = key as u32; let hihalf = (key >> 32) as u32; lohalf ^= if key >= 0 { hihalf } else { !hihalf }; Self::combine(0, Self::pg_u32_hash(lohalf)) as usize % self.shards } + /// Example of a hashing function based on SHA1. + fn sha1(&self, key: i64) -> usize { + let mut hasher = Sha1::new(); + + hasher.update(&key.to_string().as_bytes()); + + let result = hasher.finalize(); + + // Convert the SHA1 hash into hex so we can parse it as a large integer. + let hex = format!("{:x}", result); + + // Parse the last 8 bytes as an integer (8 bytes = bigint). + let key = i64::from_str_radix(&hex[hex.len() - 8..], 16).unwrap() as usize; + + key % self.shards + } + #[inline] fn rot(x: u32, k: u32) -> u32 { (x << k) | (x >> (32 - k)) @@ -109,36 +145,51 @@ mod test { // confirming that we implemented Postgres BIGINT hashing correctly. #[test] fn test_pg_bigint_hash() { - let sharder = Sharder::new(5); + let sharder = Sharder::new(5, ShardingFunction::PgBigintHash); let shard_0 = vec![1, 4, 5, 14, 19, 39, 40, 46, 47, 53]; for v in shard_0 { - assert_eq!(sharder.pg_bigint_hash(v), 0); + assert_eq!(sharder.shard(v), 0); } let shard_1 = vec![2, 3, 11, 17, 21, 23, 30, 49, 51, 54]; for v in shard_1 { - assert_eq!(sharder.pg_bigint_hash(v), 1); + assert_eq!(sharder.shard(v), 1); } let shard_2 = vec![6, 7, 15, 16, 18, 20, 25, 28, 34, 35]; for v in shard_2 { - assert_eq!(sharder.pg_bigint_hash(v), 2); + assert_eq!(sharder.shard(v), 2); } let shard_3 = vec![8, 12, 13, 22, 29, 31, 33, 36, 41, 43]; for v in shard_3 { - assert_eq!(sharder.pg_bigint_hash(v), 3); + assert_eq!(sharder.shard(v), 3); } let shard_4 = vec![9, 10, 24, 26, 27, 32, 37, 38, 42, 45]; for v in shard_4 { - assert_eq!(sharder.pg_bigint_hash(v), 4); + assert_eq!(sharder.shard(v), 4); + } + } + + #[test] + fn test_sha1_hash() { + let sharder = Sharder::new(12, ShardingFunction::Sha1); + let ids = vec![ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + ]; + let shards = vec![ + 4, 7, 8, 3, 6, 0, 0, 10, 3, 11, 1, 7, 4, 4, 11, 2, 5, 0, 8, 3, + ]; + + for (i, id) in ids.iter().enumerate() { + assert_eq!(sharder.shard(*id), shards[i]); } } } From 0ad9ab89cec6ff9efd89a1ac61de5e22a08c5bd8 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 24 Feb 2022 08:44:41 -0800 Subject: [PATCH 111/551] Switch to parking_lot RwLock & Mutex. Use trace! for protocol instead of debug! (#42) * RwLock & parking_lot::Mutex * upgrade to trace --- Cargo.lock | 1 + Cargo.toml | 1 + src/client.rs | 22 +++++++++++----------- src/main.rs | 3 ++- src/messages.rs | 2 +- src/pool.rs | 33 +++++++++++++++++++++++---------- src/server.rs | 27 +++++++++------------------ 7 files changed, 48 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1259b4e94..d6e42cb7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,6 +364,7 @@ dependencies = [ "md-5", "num_cpus", "once_cell", + "parking_lot", "rand", "regex", "serde", diff --git a/Cargo.toml b/Cargo.toml index 860955e93..d070c61da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,4 @@ sqlparser = "0.14" log = "0.4" arc-swap = "1" env_logger = "0.9" +parking_lot = "0.11" diff --git a/src/client.rs b/src/client.rs index 096125305..3c1eeeae0 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,7 +2,7 @@ /// We are pretending to the server in this scenario, /// and this module implements that. use bytes::{Buf, BufMut, BytesMut}; -use log::{debug, error}; +use log::{debug, error, trace}; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, @@ -70,7 +70,7 @@ impl Client { let transaction_mode = config.general.pool_mode.starts_with("t"); drop(config); loop { - debug!("Waiting for StartupMessage"); + trace!("Waiting for StartupMessage"); // Could be StartupMessage or SSLRequest // which makes this variable length. @@ -93,7 +93,7 @@ impl Client { match code { // Client wants SSL. We don't support it at the moment. SSL_REQUEST_CODE => { - debug!("Rejecting SSLRequest"); + trace!("Rejecting SSLRequest"); let mut no = BytesMut::with_capacity(1); no.put_u8(b'N'); @@ -103,7 +103,7 @@ impl Client { // Regular startup message. PROTOCOL_VERSION_NUMBER => { - debug!("Got StartupMessage"); + trace!("Got StartupMessage"); // TODO: perform actual auth. let parameters = parse_startup(bytes.clone())?; @@ -116,7 +116,7 @@ impl Client { write_all(&mut stream, server_info).await?; backend_key_data(&mut stream, process_id, secret_key).await?; ready_for_query(&mut stream).await?; - debug!("Startup OK"); + trace!("Startup OK"); // Split the read and write streams // so we can control buffering. @@ -168,10 +168,10 @@ impl Client { pub async fn handle(&mut self, mut pool: ConnectionPool) -> Result<(), Error> { // The client wants to cancel a query it has issued previously. if self.cancel_mode { - debug!("Sending CancelRequest"); + trace!("Sending CancelRequest"); let (process_id, secret_key, address, port) = { - let guard = self.client_server_map.lock().unwrap(); + let guard = self.client_server_map.lock(); match guard.get(&(self.process_id, self.secret_key)) { // Drop the mutex as soon as possible. @@ -202,7 +202,7 @@ impl Client { // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocols. loop { - debug!("Client idle, waiting for message"); + trace!("Client idle, waiting for message"); // Client idle, waiting for messages. self.stats.client_idle(self.process_id); @@ -216,7 +216,7 @@ impl Client { // Avoid taking a server if the client just wants to disconnect. if message[0] as char == 'X' { - debug!("Client disconnecting"); + trace!("Client disconnecting"); return Ok(()); } @@ -333,7 +333,7 @@ impl Client { let code = message.get_u8() as char; let _len = message.get_i32() as usize; - debug!("Message: {}", code); + trace!("Message: {}", code); match code { // ReadyForQuery @@ -514,7 +514,7 @@ impl Client { /// Release the server from being mine. I can't cancel its queries anymore. pub fn release(&self) { - let mut guard = self.client_server_map.lock().unwrap(); + let mut guard = self.client_server_map.lock(); guard.remove(&(self.process_id, self.secret_key)); } } diff --git a/src/main.rs b/src/main.rs index 21da9aff9..2cf67737e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,6 +36,7 @@ extern crate tokio; extern crate toml; use log::{error, info}; +use parking_lot::Mutex; use tokio::net::TcpListener; use tokio::{ signal, @@ -44,7 +45,7 @@ use tokio::{ }; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; mod client; mod config; diff --git a/src/messages.rs b/src/messages.rs index 16b2f84c3..e48af5314 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -38,7 +38,7 @@ pub async fn backend_key_data( Ok(write_all(stream, key_data).await?) } -#[allow(dead_code)] +/// Construct a `Q`: Query message. pub fn simple_query(query: &str) -> BytesMut { let mut res = BytesMut::from(&b"Q"[..]); let query = format!("{}\0", query); diff --git a/src/pool.rs b/src/pool.rs index fb33fd851..b1dda4fb2 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -3,7 +3,8 @@ use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection}; use bytes::BytesMut; use chrono::naive::NaiveDateTime; -use log::{error, info, warn}; +use log::{debug, error, info, warn}; +use parking_lot::{Mutex, RwLock}; use crate::config::{get_config, Address, Role, User}; use crate::errors::Error; @@ -11,11 +12,11 @@ use crate::server::Server; use crate::stats::Reporter; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::Instant; // Banlist: bad servers go in here. -pub type BanList = Arc>>>; +pub type BanList = Arc>>>; pub type ClientServerMap = Arc>>; #[derive(Clone, Debug)] @@ -101,7 +102,7 @@ impl ConnectionPool { databases: shards, addresses: addresses, round_robin: rand::random::() % address_len, // Start at a random replica - banlist: Arc::new(Mutex::new(banlist)), + banlist: Arc::new(RwLock::new(banlist)), stats: stats, } } @@ -161,6 +162,8 @@ impl ConnectionPool { _ => addresses.len(), }; + debug!("Allowed attempts for {:?}: {}", role, allowed_attempts); + let exists = match role { Some(role) => addresses.iter().filter(|addr| addr.role == role).count() > 0, None => true, @@ -251,14 +254,14 @@ impl ConnectionPool { pub fn ban(&self, address: &Address, shard: usize) { error!("Banning {:?}", address); let now = chrono::offset::Utc::now().naive_utc(); - let mut guard = self.banlist.lock().unwrap(); + let mut guard = self.banlist.write(); guard[shard].insert(address.clone(), now); } /// Clear the replica to receive traffic again. Takes effect immediately /// for all new transactions. pub fn _unban(&self, address: &Address, shard: usize) { - let mut guard = self.banlist.lock().unwrap(); + let mut guard = self.banlist.write(); guard[shard].remove(address); } @@ -274,12 +277,14 @@ impl ConnectionPool { Some(Role::Primary) => return false, // Primary cannot be banned. }; - // If you're not asking for the primary, - // all databases are treated as replicas. - let mut guard = self.banlist.lock().unwrap(); + debug!("Available targets for {:?}: {}", role, replicas_available); + + let guard = self.banlist.read(); // Everything is banned = nothing is banned. if guard[shard].len() == replicas_available { + drop(guard); + let mut guard = self.banlist.write(); guard[shard].clear(); drop(guard); warn!("Unbanning all replicas."); @@ -291,16 +296,24 @@ impl ConnectionPool { Some(timestamp) => { let now = chrono::offset::Utc::now().naive_utc(); let config = get_config(); + // Ban expired. if now.timestamp() - timestamp.timestamp() > config.general.ban_time { + drop(guard); + warn!("Unbanning {:?}", address); + let mut guard = self.banlist.write(); guard[shard].remove(address); false } else { + debug!("{:?} is banned", address); true } } - None => false, + None => { + debug!("{:?} is ok", address); + false + } } } diff --git a/src/server.rs b/src/server.rs index f186c0c75..934fc9534 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,7 +1,7 @@ use bytes::{Buf, BufMut, BytesMut}; ///! Implementation of the PostgreSQL server (database) protocol. ///! Here we are pretending to the a Postgres client. -use log::{debug, error, info}; +use log::{debug, error, info, trace}; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, @@ -75,7 +75,7 @@ impl Server { } }; - debug!("Sending StartupMessage"); + trace!("Sending StartupMessage"); // Send the startup packet telling the server we're a normal Postgres client. startup(&mut stream, &user.name, database).await?; @@ -97,7 +97,7 @@ impl Server { Err(_) => return Err(Error::SocketError), }; - debug!("Message: {}", code); + trace!("Message: {}", code); match code { // Authentication @@ -108,7 +108,7 @@ impl Server { Err(_) => return Err(Error::SocketError), }; - debug!("Auth: {}", auth_code); + trace!("Auth: {}", auth_code); match auth_code { MD5_ENCRYPTED_PASSWORD => { @@ -141,7 +141,7 @@ impl Server { Err(_) => return Err(Error::SocketError), }; - debug!("Error: {}", error_code); + trace!("Error: {}", error_code); match error_code { // No error message is present in the message. @@ -300,7 +300,7 @@ impl Server { let code = message.get_u8() as char; let _len = message.get_i32(); - debug!("Message: {}", code); + trace!("Message: {}", code); match code { // ReadyForQuery @@ -415,7 +415,7 @@ impl Server { /// Claim this server as mine for the purposes of query cancellation. pub fn claim(&mut self, process_id: i32, secret_key: i32) { - let mut guard = self.client_server_map.lock().unwrap(); + let mut guard = self.client_server_map.lock(); guard.insert( (process_id, secret_key), ( @@ -431,18 +431,9 @@ impl Server { /// It will use the simple query protocol. /// Result will not be returned, so this is useful for things like `SET` or `ROLLBACK`. pub async fn query(&mut self, query: &str) -> Result<(), Error> { - let mut query = BytesMut::from(&query.as_bytes()[..]); - query.put_u8(0); // C-string terminator (NULL character). + let query = simple_query(query); - let len = query.len() as i32 + 4; - - let mut msg = BytesMut::with_capacity(len as usize + 1); - - msg.put_u8(b'Q'); - msg.put_i32(len); - msg.put_slice(&query[..]); - - self.send(msg).await?; + self.send(query).await?; loop { let _ = self.recv().await?; From ce8a76c49752aa355154e9b09e94c418884178e0 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 24 Feb 2022 12:16:24 -0800 Subject: [PATCH 112/551] Allow to set shard and set sharding key without quotes (#43) * Allow to set shard and set sharding key without quotes * cover it * dont look for these in the middle of another query * friendly regex * its own response to set shard key --- src/client.rs | 8 ++++- src/query_router.rs | 82 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 78 insertions(+), 12 deletions(-) diff --git a/src/client.rs b/src/client.rs index 3c1eeeae0..fb1997d50 100644 --- a/src/client.rs +++ b/src/client.rs @@ -229,11 +229,17 @@ impl Client { } } - Some((Command::SetShard, _)) | Some((Command::SetShardingKey, _)) => { + Some((Command::SetShard, _)) => { custom_protocol_response_ok(&mut self.write, &format!("SET SHARD")).await?; continue; } + Some((Command::SetShardingKey, _)) => { + custom_protocol_response_ok(&mut self.write, &format!("SET SHARDING KEY")) + .await?; + continue; + } + Some((Command::SetServerRole, _)) => { custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; continue; diff --git a/src/query_router.rs b/src/query_router.rs index 3f309ea7b..737c4d13f 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -5,17 +5,17 @@ use crate::sharding::{Sharder, ShardingFunction}; use bytes::{Buf, BytesMut}; use log::{debug, error}; use once_cell::sync::OnceCell; -use regex::RegexSet; +use regex::{Regex, RegexSet}; use sqlparser::ast::Statement::{Query, StartTransaction}; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; const CUSTOM_SQL_REGEXES: [&str; 5] = [ - r"(?i)SET SHARDING KEY TO '[0-9]+'", - r"(?i)SET SHARD TO '[0-9]+'", - r"(?i)SHOW SHARD", - r"(?i)SET SERVER ROLE TO '(PRIMARY|REPLICA|ANY|AUTO|DEFAULT)'", - r"(?i)SHOW SERVER ROLE", + r"(?i)^ *SET SHARDING KEY TO '?([0-9]+)'? *;? *$", + r"(?i)^ *SET SHARD TO '?([0-9]+)'? *;? *$", + r"(?i)^ *SHOW SHARD *;? *$", + r"(?i)^ *SET SERVER ROLE TO '(PRIMARY|REPLICA|ANY|AUTO|DEFAULT)' *;? *$", + r"(?i)^ *SHOW SERVER ROLE *;? *$", ]; #[derive(PartialEq, Debug)] @@ -27,8 +27,12 @@ pub enum Command { ShowServerRole, } +// Quick test static CUSTOM_SQL_REGEX_SET: OnceCell = OnceCell::new(); +// Capture value +static CUSTOM_SQL_REGEX_LIST: OnceCell> = OnceCell::new(); + pub struct QueryRouter { // By default, queries go here, unless we have better information // about what the client wants. @@ -63,6 +67,21 @@ impl QueryRouter { } }; + let list: Vec<_> = CUSTOM_SQL_REGEXES + .iter() + .map(|rgx| Regex::new(rgx).unwrap()) + .collect(); + + // Impossible + if list.len() != set.len() { + return false; + } + + match CUSTOM_SQL_REGEX_LIST.set(list) { + Ok(_) => true, + Err(_) => return false, + }; + match CUSTOM_SQL_REGEX_SET.set(set) { Ok(_) => true, Err(_) => false, @@ -113,6 +132,11 @@ impl QueryRouter { None => return None, }; + let regex_list = match CUSTOM_SQL_REGEX_LIST.get() { + Some(regex_list) => regex_list, + None => return None, + }; + let matches: Vec<_> = regex_set.matches(&query).into_iter().collect(); if matches.len() != 1 { @@ -130,7 +154,19 @@ impl QueryRouter { let mut value = match command { Command::SetShardingKey | Command::SetShard | Command::SetServerRole => { - query.split("'").collect::>()[1].to_string() + // Capture value. I know this re-runs the regex engine, but I haven't + // figured out a better way just yet. I think I can write a single Regex + // that matches all 5 custom SQL patterns, but maybe that's not very legible? + // + // I think this is faster than running the Regex engine 5 times, so + // this is a strong maybe for me so far. + match regex_list[matches[0]].captures(&query) { + Some(captures) => match captures.get(1) { + Some(value) => value.as_str().to_string(), + None => return None, + }, + None => return None, + } } Command::ShowShard => self.shard().to_string(), @@ -411,14 +447,38 @@ mod test { "set server role to 'any'", "set server role to 'auto'", "show server role", + // No quotes + "SET SHARDING KEY TO 11235", + "SET SHARD TO 15", + // Spaces and semicolon + " SET SHARDING KEY TO 11235 ; ", + " SET SHARD TO 15; ", + " SET SHARDING KEY TO 11235 ;", + " SET SERVER ROLE TO 'primary'; ", + " SET SERVER ROLE TO 'primary' ; ", + " SET SERVER ROLE TO 'primary' ;", + ]; + + // Which regexes it'll match to in the list + let matches = [ + 0, 1, 2, 3, 3, 3, 3, 4, 0, 1, 2, 3, 3, 3, 3, 4, 0, 1, 0, 1, 0, 3, 3, 3, ]; + let list = CUSTOM_SQL_REGEX_LIST.get().unwrap(); let set = CUSTOM_SQL_REGEX_SET.get().unwrap(); - for test in &tests { - let matches: Vec<_> = set.matches(test).into_iter().collect(); + for (i, test) in tests.iter().enumerate() { + assert!(list[matches[i]].is_match(test)); + assert_eq!(set.matches(test).into_iter().collect::>().len(), 1); + } + + let bad = [ + "SELECT * FROM table", + "SELECT * FROM table WHERE value = 'set sharding key to 5'", // Don't capture things in the middle of the query + ]; - assert_eq!(matches.len(), 1); + for query in &bad { + assert_eq!(set.matches(query).into_iter().collect::>().len(), 0); } } @@ -428,7 +488,7 @@ mod test { let mut qr = QueryRouter::new(); // SetShardingKey - let query = simple_query("SET SHARDING KEY TO '13'"); + let query = simple_query("SET SHARDING KEY TO 13"); assert_eq!( qr.try_execute_command(query), Some((Command::SetShardingKey, String::from("1"))) From bd5cc0c2dd64251f8581b9164bf2d93bb3d8976d Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 24 Feb 2022 20:55:19 -0800 Subject: [PATCH 113/551] Use Toxiproxy for failover testing (#44) * Toxiproxy * up-to-date config * debug * hm * more * mroe * more * hmm * aha * less logs * cleaner * hmm * we test these now * update readme --- .circleci/pgcat.toml | 108 +++++++++++++++++++++++++++++++++++++++++ .circleci/run_tests.sh | 61 +++++++++++++++++------ README.md | 4 +- src/client.rs | 2 +- 4 files changed, 158 insertions(+), 17 deletions(-) create mode 100644 .circleci/pgcat.toml diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml new file mode 100644 index 000000000..370b06b18 --- /dev/null +++ b/.circleci/pgcat.toml @@ -0,0 +1,108 @@ +# +# PgCat config example. +# + +# +# General pooler settings +[general] + +# What IP to run on, 0.0.0.0 means accessible from everywhere. +host = "0.0.0.0" + +# Port to run on, same as PgBouncer used in this example. +port = 6432 + +# How many connections to allocate per server. +pool_size = 15 + +# Pool mode (see PgBouncer docs for more). +# session: one server connection per connected client +# transaction: one server connection per client transaction +pool_mode = "transaction" + +# How long to wait before aborting a server connection (ms). +connect_timeout = 100 + +# How much time to give `SELECT 1` health check query to return with a result (ms). +healthcheck_timeout = 100 + +# For how long to ban a server if it fails a health check (seconds). +ban_time = 60 # Seconds + +# Stats will be sent here +statsd_address = "127.0.0.1:8125" + +# +# User to use for authentication against the server. +[user] +name = "sharding_user" +password = "sharding_user" + + +# +# Shards in the cluster +[shards] + +# Shard 0 +[shards.0] + +# [ host, port, role ] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5433, "replica" ], + # [ "127.0.1.1", 5432, "replica" ], +] +# Database name (e.g. "postgres") +database = "shard0" + +[shards.1] +# [ host, port, role ] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5433, "replica" ], + # [ "127.0.1.1", 5432, "replica" ], +] +database = "shard1" + +[shards.2] +# [ host, port, role ] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5433, "replica" ], + # [ "127.0.1.1", 5432, "replica" ], +] +database = "shard2" + + +# Settings for our query routing layer. +[query_router] + +# If the client doesn't specify, route traffic to +# this role by default. +# +# any: round-robin between primary and replicas, +# replica: round-robin between replicas only without touching the primary, +# primary: all queries go to the primary unless otherwise specified. +default_role = "any" + + +# Query parser. If enabled, we'll attempt to parse +# every incoming query to determine if it's a read or a write. +# If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, +# we'll direct it to the primary. +query_parser_enabled = false + +# If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for +# load balancing of read queries. Otherwise, the primary will only be used for write +# queries. The primary can always be explicitely selected with our custom protocol. +primary_reads_enabled = true + +# So what if you wanted to implement a different hashing function, +# or you've already built one and you want this pooler to use it? +# +# Current options: +# +# pg_bigint_hash: PARTITION BY HASH (Postgres hashing function) +# sha1: A hashing function based on SHA1 +# +sharding_function = "pg_bigint_hash" diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 2204b241f..73a237bef 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -3,20 +3,34 @@ set -e set -o xtrace +# Start PgCat with a particular log level +# for inspection. +function start_pgcat() { + kill -s SIGINT $(pgrep pgcat) || true + RUST_LOG=${1} ./target/debug/pgcat .circleci/pgcat.toml & + sleep 1 +} + +# Setup the database with shards and user psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql -./target/debug/pgcat & +# Install Toxiproxy to simulate a downed/slow database +wget -O toxiproxy-2.1.4.deb https://github.com/Shopify/toxiproxy/releases/download/v2.1.4/toxiproxy_2.1.4_amd64.deb +sudo dpkg -i toxiproxy-2.1.4.deb +# Start Toxiproxy +toxiproxy-server & sleep 1 -# Setup PgBench -pgbench -i -h 127.0.0.1 -p 6432 +# Create a database at port 5433, forward it to Postgres +toxiproxy-cli create -l 127.0.0.1:5433 -u 127.0.0.1:5432 postgres_replica -# Run it -pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol simple +start_pgcat "info" -# Extended protocol -pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended +# pgbench test +pgbench -i -h 127.0.0.1 -p 6432 && \ + pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol simple && \ + pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended # COPY TO STDOUT test psql -h 127.0.0.1 -p 6432 -c 'COPY (SELECT * FROM pgbench_accounts LIMIT 15) TO STDOUT;' > /dev/null @@ -35,18 +49,37 @@ psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql > / psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null # -# ActiveRecord tests! +# ActiveRecord tests # -cd tests/ruby -sudo gem install bundler -bundle install -ruby tests.rb +cd tests/ruby && \ + sudo gem install bundler && \ + bundle install && \ + ruby tests.rb && \ +cd ../.. + +# Start PgCat in debug to demonstrate failover better +start_pgcat "debug" + +# Add latency to the replica at port 5433 slightly above the healthcheck timeout +toxiproxy-cli toxic add -t latency -a latency=300 postgres_replica +sleep 1 + +# Note the failover in the logs +timeout 5 psql -e -h 127.0.0.1 -p 6432 <<-EOF +SELECT 1; +SELECT 1; +SELECT 1; +EOF + +# Remove latency +toxiproxy-cli toxic remove --toxicName latency_downstream postgres_replica + +start_pgcat "info" -cd ../../ # Test session mode (and config reload) sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' pgcat.toml -# Reload config +# Reload config test kill -SIGHUP $(pgrep pgcat) # Prepared statements that will only work in session mode diff --git a/README.md b/README.md index b05eaafd5..2cc758662 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,8 @@ See [sharding README](./tests/sharding/README.md) for sharding logic testing. | Session pooling | :white_check_mark: | :white_check_mark: | Tested by running pgbench with `--protocol prepared` which only works in session mode. | | `COPY` | :white_check_mark: | :white_check_mark: | `pgbench -i` uses `COPY`. `COPY FROM` is tested as well. | | Query cancellation | :white_check_mark: | :white_check_mark: | `psql -c 'SELECT pg_sleep(1000);'` and press `Ctrl-C`. | -| Load balancing | :x: | :white_check_mark: | We could test this by emitting statistics for each replica and compare them. | -| Failover | :x: | :white_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing could include using Toxiproxy. | +| Load balancing | :white_check_mark: | :white_check_mark: | We could test this by emitting statistics for each replica and compare them. | +| Failover | :white_check_mark: | :white_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing is using Toxiproxy. | | Sharding | :white_check_mark: | :white_check_mark: | See `tests/sharding` and `tests/ruby` for an Rails/ActiveRecord example. | | Statistics reporting | :x: | :white_check_mark: | Run `nc -l -u 8125` and watch the stats come in every 15 seconds. | | Live config reloading | :white_check_mark: | :white_check_mark: | Run `kill -s SIGHUP $(pgrep pgcat)` and watch the config reload. | diff --git a/src/client.rs b/src/client.rs index fb1997d50..b0ca7d1da 100644 --- a/src/client.rs +++ b/src/client.rs @@ -311,7 +311,7 @@ impl Client { // or until the client disconnects if we are in session mode. loop { let mut message = if message.len() == 0 { - debug!("Waiting for message inside transaction or in session mode"); + trace!("Waiting for message inside transaction or in session mode"); match read_message(&mut self.read).await { Ok(message) => message, From 11cc513f200fd5f82d76b252d8bcc7363501d293 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 25 Feb 2022 14:27:33 -0800 Subject: [PATCH 114/551] query routing docs (#45) --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index 2cc758662..ab9406167 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,31 @@ All servers are checked with a `SELECT 1` query before being given to a client. The ban time can be changed with `ban_time`. The default is 60 seconds. +Failover behavior can get pretty interesting (read complex) when multiple configurations and factors are involved. The table below will try to explain what PgCat does in each scenario: + +| **Query** | **`SET SERVER ROLE TO`** | **`query_parser_enabled`** | **`primary_reads_enabled`** | **Target state** | **Outcome** | +|---------------------------|--------------------------|----------------------------|-----------------------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Read query, i.e. `SELECT` | unset (any) | false | false | up | Query is routed to the first instance in the round-robin loop. | +| Read query | unset (any) | true | false | up | Query is routed to the first replica instance in the round-robin loop. | +| Read query | unset (any) | true | true | up | Query is routed to the first instance in the round-robin loop. | +| Read query | replica | false | false | up | Query is routed to the first replica instance in the round-robin loop. | +| Read query | primary | false | false | up | Query is routed to the primary. | +| Read query | unset (any) | false | false | down | First instance is banned for reads. Next target in the round-robin loop is attempted. | +| Read query | unset (any) | true | false | down | First replica instance is banned. Next replica instance is attempted in the round-robin loop. | +| Read query | unset (any) | true | true | down | First instance (even if primary) is banned for reads. Next instance is attempted in the round-robin loop. | +| Read query | replica | false | false | down | First replica instance is banned. Next replica instance is attempted in the round-robin loop. | +| Read query | primary | false | false | down | The query is attempted against the primary and fails. The client receives an error. | +| | | | | | | +| Write query e.g. `INSERT` | unset (any) | false | false | up | The query is attempted against the first available instance in the round-robin loop. If the instance is a replica, the query fails and the client receives an error. | +| Write query | unset (any) | true | false | up | The query is routed to the primary. | +| Write query | unset (any) | true | true | up | The query is routed to the primary. | +| Write query | primary | false | false | up | The query is routed to the primary. | +| Write query | replica | false | false | up | The query is routed to the replica and fails. The client receives an error. | +| Write query | unset (any) | true | false | down | The query is routed to the primary and fails. The client receives an error. | +| Write query | unset (any) | true | true | down | The query is routed to the primary and fails. The client receives an error. | +| Write query | primary | false | false | down | The query is routed to the primary and fails. The client receives an error. | +| | | | | | | + ### Sharding We use the `PARTITION BY HASH` hashing function, the same as used by Postgres for declarative partitioning. This allows to shard the database using Postgres partitions and place the partitions on different servers (shards). Both read and write queries can be routed to the shards using this pooler. From 22931b0d1d6050419927ac8a8d9fd99d6511e833 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 25 Feb 2022 18:20:15 -0800 Subject: [PATCH 115/551] admin: SHOW STATS (#46) * admin: show stats * warning * tests * lint * type mod --- .circleci/run_tests.sh | 4 ++ src/admin.rs | 116 +++++++++++++++++++++++++++++++++++++++++ src/client.rs | 22 ++++++++ src/constants.rs | 6 +++ src/main.rs | 1 + src/messages.rs | 2 +- src/stats.rs | 35 ++++++++++++- 7 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 src/admin.rs diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 73a237bef..93827f9ae 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -57,6 +57,10 @@ cd tests/ruby && \ ruby tests.rb && \ cd ../.. +# Admin tests +psql -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/null +(! psql -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) + # Start PgCat in debug to demonstrate failover better start_pgcat "debug" diff --git a/src/admin.rs b/src/admin.rs new file mode 100644 index 000000000..5108346c6 --- /dev/null +++ b/src/admin.rs @@ -0,0 +1,116 @@ +use bytes::{Buf, BufMut, BytesMut}; +use log::trace; +use tokio::net::tcp::OwnedWriteHalf; + +use std::collections::HashMap; + +use crate::constants::{OID_NUMERIC, OID_TEXT}; +use crate::errors::Error; +use crate::messages::write_all_half; +use crate::stats::get_stats; + +/// Handle admin client +pub async fn handle_admin(stream: &mut OwnedWriteHalf, mut query: BytesMut) -> Result<(), Error> { + let code = query.get_u8() as char; + + if code != 'Q' { + return Err(Error::ProtocolSyncError); + } + + let len = query.get_i32() as usize; + let query = String::from_utf8_lossy(&query[..len - 5]) + .to_string() + .to_ascii_uppercase(); + + if query.starts_with("SHOW STATS") { + trace!("SHOW STATS"); + show_stats(stream).await + } else { + Err(Error::ProtocolSyncError) + } +} + +/// SHOW STATS +pub async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { + let columns = [ + "database", + "total_xact_count", + "total_query_count", + "total_received", + "total_sent", + "total_xact_time", + "total_query_time", + "total_wait_time", + "avg_xact_count", + "avg_query_count", + "avg_recv", + "avg_sent", + "avg_xact_time", + "avg_query_time", + "avg_wait_time", + ]; + + let stats = get_stats().unwrap_or(HashMap::new()); + let mut res = BytesMut::new(); + let mut row_desc = BytesMut::new(); + let mut data_row = BytesMut::new(); + + // Number of columns: 1 + row_desc.put_i16(columns.len() as i16); + data_row.put_i16(columns.len() as i16); + + for (i, column) in columns.iter().enumerate() { + // RowDescription + + // Column name + row_desc.put_slice(&format!("{}\0", column).as_bytes()); + + // Doesn't belong to any table + row_desc.put_i32(0); + + // Doesn't belong to any table + row_desc.put_i16(0); + + // Data type + row_desc.put_i32(if i == 0 { OID_TEXT } else { OID_NUMERIC }); + + // Numeric/text size = variable (-1) + row_desc.put_i16(-1); + + // Type modifier: none that I know + row_desc.put_i32(-1); + + // Format being used: text (0), binary (1) + row_desc.put_i16(0); + + // DataRow + let value = if i == 0 { + String::from("all shards") + } else { + stats.get(&column.to_string()).unwrap_or(&0).to_string() + }; + + data_row.put_i32(value.len() as i32); + data_row.put_slice(value.as_bytes()); + } + + let command_complete = BytesMut::from(&"SHOW\0"[..]); + + res.put_u8(b'T'); + res.put_i32(row_desc.len() as i32 + 4); + res.put(row_desc); + + res.put_u8(b'D'); + res.put_i32(data_row.len() as i32 + 4); + res.put(data_row); + + res.put_u8(b'C'); + res.put_i32(command_complete.len() as i32 + 4); + res.put(command_complete); + + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} diff --git a/src/client.rs b/src/client.rs index b0ca7d1da..fb1a4b0ec 100644 --- a/src/client.rs +++ b/src/client.rs @@ -11,6 +11,7 @@ use tokio::net::{ use std::collections::HashMap; +use crate::admin::handle_admin; use crate::config::get_config; use crate::constants::*; use crate::errors::Error; @@ -54,6 +55,9 @@ pub struct Client { // Statistics stats: Reporter, + + // Clients want to talk to admin + admin: bool, } impl Client { @@ -118,6 +122,15 @@ impl Client { ready_for_query(&mut stream).await?; trace!("Startup OK"); + let database = parameters + .get("database") + .unwrap_or(parameters.get("user").unwrap()); + let admin = ["pgcat", "pgbouncer"] + .iter() + .filter(|db| *db == &database) + .count() + == 1; + // Split the read and write streams // so we can control buffering. let (read, write) = stream.into_split(); @@ -133,6 +146,7 @@ impl Client { client_server_map: client_server_map, parameters: parameters, stats: stats, + admin: admin, }); } @@ -154,6 +168,7 @@ impl Client { client_server_map: client_server_map, parameters: HashMap::new(), stats: stats, + admin: false, }); } @@ -220,6 +235,13 @@ impl Client { return Ok(()); } + // Handle admin database real quick + if self.admin { + trace!("Handling admin command"); + handle_admin(&mut self.write, message).await?; + continue; + } + // Handle all custom protocol commands here. match query_router.try_execute_command(message.clone()) { // Normal query diff --git a/src/constants.rs b/src/constants.rs index 3bdb47d8e..25609759a 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -20,3 +20,9 @@ pub const AUTHENTICATION_SUCCESSFUL: i32 = 0; // ErrorResponse: A code identifying the field type; if zero, this is the message terminator and no string follows. pub const MESSAGE_TERMINATOR: u8 = 0; + +// +// Data types +// +pub const OID_NUMERIC: i32 = 1700; +pub const OID_TEXT: i32 = 25; diff --git a/src/main.rs b/src/main.rs index 2cf67737e..a41d5ecc6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,6 +47,7 @@ use tokio::{ use std::collections::HashMap; use std::sync::Arc; +mod admin; mod client; mod config; mod constants; diff --git a/src/messages.rs b/src/messages.rs index e48af5314..e24ad9d7a 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -274,7 +274,7 @@ pub async fn show_response( row_desc.put_i16(-1); // Type modifier: none that I know - row_desc.put_i32(0); + row_desc.put_i32(-1); // Format being used: text (0), binary (1) row_desc.put_i16(0); diff --git a/src/stats.rs b/src/stats.rs index 384aa1a26..40a8ab756 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,12 +1,16 @@ -use log::info; +use log::{error, info}; +use once_cell::sync::OnceCell; use statsd::Client; /// Events collector and publisher. use tokio::sync::mpsc::{Receiver, Sender}; use std::collections::HashMap; +use std::sync::{Arc, Mutex}; use crate::config::get_config; +static LATEST_STATS: OnceCell>>> = OnceCell::new(); + #[derive(Debug, Clone, Copy)] enum EventName { CheckoutTime, @@ -212,6 +216,13 @@ impl Collector { pub async fn collect(&mut self) { info!("Events reporter started"); + match LATEST_STATS.set(Arc::new(Mutex::new(HashMap::new()))) { + Ok(_) => (), + Err(_) => { + error!("Latest stats will not be available"); + } + }; + let mut stats = HashMap::from([ ("total_query_count", 0), ("total_xact_count", 0), @@ -352,6 +363,17 @@ impl Collector { info!("{:?}", stats); + match LATEST_STATS.get() { + Some(arc) => { + let mut guard = arc.lock().unwrap(); + for (key, value) in &stats { + guard.insert(key.to_string(), value.clone()); + } + } + + None => (), + }; + let mut pipeline = self.client.pipeline(); for (key, value) in stats.iter_mut() { @@ -365,3 +387,14 @@ impl Collector { } } } + +pub fn get_stats() -> Option> { + match LATEST_STATS.get() { + Some(arc) => { + let guard = arc.lock().unwrap(); + Some(guard.clone()) + } + + None => None, + } +} From 29a85e032a7a8fd42f5a9e7cfb7ee5d3ea18cbbf Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 26 Feb 2022 10:03:11 -0800 Subject: [PATCH 116/551] Fix some stats (#47) * fix some stats * use constant * lint --- src/stats.rs | 68 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/src/stats.rs b/src/stats.rs index 40a8ab756..399bee65c 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,4 +1,4 @@ -use log::{error, info}; +use log::{debug, error, info}; use once_cell::sync::OnceCell; use statsd::Client; /// Events collector and publisher. @@ -10,6 +10,7 @@ use std::sync::{Arc, Mutex}; use crate::config::get_config; static LATEST_STATS: OnceCell>>> = OnceCell::new(); +static STAT_PERIOD: u64 = 15000; //15 seconds #[derive(Debug, Clone, Copy)] enum EventName { @@ -228,7 +229,15 @@ impl Collector { ("total_xact_count", 0), ("total_sent", 0), ("total_received", 0), + ("total_xact_time", 0), + ("total_query_time", 0), ("total_wait_time", 0), + ("avg_xact_time", 0), + ("avg_query_time", 0), + ("avg_xact_count", 0), + ("avg_sent", 0), + ("avg_received", 0), + ("avg_wait_time", 0), ("maxwait_us", 0), ("maxwait", 0), ("cl_waiting", 0), @@ -240,11 +249,18 @@ impl Collector { ("sv_tested", 0), ]); + // Stats saved after each iteration of the flush event. Used in calculation + // of averages in the last flush period. + let mut old_stats: HashMap = HashMap::new(); + + // Track which state the client and server are at any given time. let mut client_server_states: HashMap = HashMap::new(); - let tx = self.tx.clone(); + // Flush stats to StatsD and calculate averages every 15 seconds. + let tx = self.tx.clone(); tokio::task::spawn(async move { - let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(15000)); + let mut interval = + tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD)); loop { interval.tick().await; let _ = tx.try_send(Event { @@ -255,6 +271,7 @@ impl Collector { } }); + // The collector loop loop { let stat = match self.rx.recv().await { Some(stat) => stat, @@ -291,10 +308,11 @@ impl Collector { *counter += stat.value; let counter = stats.entry("maxwait_us").or_insert(0); + let mic_part = stat.value % 1_000_000; // Report max time here - if stat.value > *counter { - *counter = stat.value; + if mic_part > *counter { + *counter = mic_part; } let counter = stats.entry("maxwait").or_insert(0); @@ -320,6 +338,7 @@ impl Collector { } EventName::FlushStatsToStatsD => { + // Calculate connection states for (_, state) in &client_server_states { match state { EventName::ClientActive => { @@ -361,8 +380,26 @@ impl Collector { }; } - info!("{:?}", stats); + // Calculate averages + for stat in &[ + "avg_query_count", + "avgxact_count", + "avg_sent", + "avg_received", + "avg_wait_time", + ] { + let total_name = stat.replace("avg_", "total_"); + let old_value = old_stats.entry(total_name.clone()).or_insert(0); + let new_value = stats.get(total_name.as_str()).unwrap_or(&0).to_owned(); + let avg = (new_value - *old_value) / (STAT_PERIOD as i64 / 1_000); // Avg / second + + stats.insert(stat, avg); + *old_value = new_value; + } + debug!("{:?}", stats); + + // Update latest stats used in SHOW STATS match LATEST_STATS.get() { Some(arc) => { let mut guard = arc.lock().unwrap(); @@ -376,9 +413,24 @@ impl Collector { let mut pipeline = self.client.pipeline(); - for (key, value) in stats.iter_mut() { + for (key, value) in stats.iter() { pipeline.gauge(key, *value as f64); - *value = 0; + } + + // These are re-calculated every iteration of the loop, so we don't want to add values + // from the last iteration. + for stat in &[ + "cl_active", + "cl_waiting", + "cl_idle", + "sv_idle", + "sv_active", + "sv_tested", + "sv_login", + "maxwait", + "maxwait_us", + ] { + stats.insert(stat, 0); } pipeline.send(&self.client); From 897e914a4d16f3e84fa6d101eb65783260275aa0 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 26 Feb 2022 11:01:52 -0800 Subject: [PATCH 117/551] Check server versions on startup & refactor (#48) * Refactor and check server parameters * warnings * fix validator --- src/admin.rs | 4 +--- src/messages.rs | 15 ++++++++++---- src/pool.rs | 16 +++++++++++++-- src/stats.rs | 52 +++++++++++-------------------------------------- 4 files changed, 37 insertions(+), 50 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 5108346c6..a19febd27 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -2,8 +2,6 @@ use bytes::{Buf, BufMut, BytesMut}; use log::trace; use tokio::net::tcp::OwnedWriteHalf; -use std::collections::HashMap; - use crate::constants::{OID_NUMERIC, OID_TEXT}; use crate::errors::Error; use crate::messages::write_all_half; @@ -50,7 +48,7 @@ pub async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { "avg_wait_time", ]; - let stats = get_stats().unwrap_or(HashMap::new()); + let stats = get_stats(); let mut res = BytesMut::new(); let mut row_desc = BytesMut::new(); let mut data_row = BytesMut::new(); diff --git a/src/messages.rs b/src/messages.rs index e24ad9d7a..8f7080f44 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -91,9 +91,8 @@ pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Resu } } -/// Parse StartupMessage parameters. -/// e.g. user, database, application_name, etc. -pub fn parse_startup(mut bytes: BytesMut) -> Result, Error> { +/// Parse the params the server sends as a key/value format. +pub fn parse_params(mut bytes: BytesMut) -> Result, Error> { let mut result = HashMap::new(); let mut buf = Vec::new(); let mut tmp = String::new(); @@ -115,7 +114,7 @@ pub fn parse_startup(mut bytes: BytesMut) -> Result, Err // Expect pairs of name and value // and at least one pair to be present. - if buf.len() % 2 != 0 && buf.len() >= 2 { + if buf.len() % 2 != 0 || buf.len() < 2 { return Err(Error::ClientBadStartup); } @@ -127,6 +126,14 @@ pub fn parse_startup(mut bytes: BytesMut) -> Result, Err i += 2; } + Ok(result) +} + +/// Parse StartupMessage parameters. +/// e.g. user, database, application_name, etc. +pub fn parse_startup(bytes: BytesMut) -> Result, Error> { + let result = parse_params(bytes)?; + // Minimum required parameters // I want to have the user at the very minimum, according to the protocol spec. if !result.contains_key("user") { diff --git a/src/pool.rs b/src/pool.rs index b1dda4fb2..26d172bea 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -126,10 +126,22 @@ impl ConnectionPool { }; let mut proxy = connection.0; - let _address = connection.1; + let address = connection.1; let server = &mut *proxy; - server_infos.push(server.server_info()); + let server_info = server.server_info(); + + if server_infos.len() > 0 { + // Compare against the last server checked. + if server_info != server_infos[server_infos.len() - 1] { + warn!( + "{:?} has different server configuration than the last server", + address + ); + } + } + + server_infos.push(server_info); } } diff --git a/src/stats.rs b/src/stats.rs index 399bee65c..99f709dbd 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,15 +1,15 @@ -use log::{debug, error, info}; -use once_cell::sync::OnceCell; +use log::{debug, info}; +use once_cell::sync::Lazy; +use parking_lot::Mutex; use statsd::Client; -/// Events collector and publisher. use tokio::sync::mpsc::{Receiver, Sender}; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; use crate::config::get_config; -static LATEST_STATS: OnceCell>>> = OnceCell::new(); +// Stats used in SHOW STATS +static LATEST_STATS: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); static STAT_PERIOD: u64 = 15000; //15 seconds #[derive(Debug, Clone, Copy)] @@ -187,16 +187,6 @@ impl Reporter { let _ = self.tx.try_send(event); } - - // pub fn flush_to_statsd(&self) { - // let event = Event { - // name: EventName::FlushStatsToStatsD, - // value: 0, - // process_id: None, - // }; - - // let _ = self.tx.try_send(event); - // } } pub struct Collector { @@ -217,13 +207,6 @@ impl Collector { pub async fn collect(&mut self) { info!("Events reporter started"); - match LATEST_STATS.set(Arc::new(Mutex::new(HashMap::new()))) { - Ok(_) => (), - Err(_) => { - error!("Latest stats will not be available"); - } - }; - let mut stats = HashMap::from([ ("total_query_count", 0), ("total_xact_count", 0), @@ -400,16 +383,10 @@ impl Collector { debug!("{:?}", stats); // Update latest stats used in SHOW STATS - match LATEST_STATS.get() { - Some(arc) => { - let mut guard = arc.lock().unwrap(); - for (key, value) in &stats { - guard.insert(key.to_string(), value.clone()); - } - } - - None => (), - }; + let mut guard = LATEST_STATS.lock(); + for (key, value) in &stats { + guard.insert(key.to_string(), value.clone()); + } let mut pipeline = self.client.pipeline(); @@ -440,13 +417,6 @@ impl Collector { } } -pub fn get_stats() -> Option> { - match LATEST_STATS.get() { - Some(arc) => { - let guard = arc.lock().unwrap(); - Some(guard.clone()) - } - - None => None, - } +pub fn get_stats() -> HashMap { + LATEST_STATS.lock().clone() } From 7c20c16ee6b32b9777222eec9f63e999bc887487 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 27 Feb 2022 10:21:24 -0800 Subject: [PATCH 118/551] admin RELOAD (#49) * admin RELOAD * test --- .circleci/run_tests.sh | 3 +++ src/admin.rs | 35 ++++++++++++++++++++++++++++++++++- src/config.rs | 7 ++++++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 93827f9ae..abc1a7f57 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -86,6 +86,9 @@ sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' pgcat.toml # Reload config test kill -SIGHUP $(pgrep pgcat) +# Reload again with the admin database +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null + # Prepared statements that will only work in session mode pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol prepared diff --git a/src/admin.rs b/src/admin.rs index a19febd27..97346ed87 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,7 +1,8 @@ use bytes::{Buf, BufMut, BytesMut}; -use log::trace; +use log::{info, trace}; use tokio::net::tcp::OwnedWriteHalf; +use crate::config::{get_config, parse}; use crate::constants::{OID_NUMERIC, OID_TEXT}; use crate::errors::Error; use crate::messages::write_all_half; @@ -23,11 +24,43 @@ pub async fn handle_admin(stream: &mut OwnedWriteHalf, mut query: BytesMut) -> R if query.starts_with("SHOW STATS") { trace!("SHOW STATS"); show_stats(stream).await + } else if query.starts_with("RELOAD") { + trace!("RELOAD"); + reload(stream).await } else { Err(Error::ProtocolSyncError) } } +/// RELOAD +pub async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { + info!("Reloading config"); + + let config = get_config(); + let path = config.path.clone().unwrap(); + + parse(&path).await?; + + let config = get_config(); + + config.show(); + + let mut res = BytesMut::new(); + + // CommandComplete + let command_complete = BytesMut::from(&"RELOAD\0"[..]); + res.put_u8(b'C'); + res.put_i32(command_complete.len() as i32 + 4); + res.put(command_complete); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} + /// SHOW STATS pub async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let columns = [ diff --git a/src/config.rs b/src/config.rs index 754dc4fb2..3cb413a88 100644 --- a/src/config.rs +++ b/src/config.rs @@ -134,6 +134,7 @@ impl Default for QueryRouter { #[derive(Deserialize, Debug, Clone)] pub struct Config { + pub path: Option, pub general: General, pub user: User, pub shards: HashMap, @@ -143,6 +144,7 @@ pub struct Config { impl Default for Config { fn default() -> Config { Config { + path: Some(String::from("pgcat.toml")), general: General::default(), user: User::default(), shards: HashMap::from([(String::from("1"), Shard::default())]), @@ -189,7 +191,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; - let config: Config = match toml::from_str(&contents) { + let mut config: Config = match toml::from_str(&contents) { Ok(config) => config, Err(err) => { error!("Could not parse config file: {}", err.to_string()); @@ -279,6 +281,8 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; + config.path = Some(path.to_string()); + CONFIG.store(Arc::new(config.clone())); Ok(()) @@ -296,5 +300,6 @@ mod test { assert_eq!(get_config().shards["1"].servers[0].0, "127.0.0.1"); assert_eq!(get_config().shards["0"].servers[0].2, "primary"); assert_eq!(get_config().query_router.default_role, "any"); + assert_eq!(get_config().path, Some("pgcat.toml".to_string())); } } From 67f558a41ee5dc6bcb33bad6bd52ba27a59e3671 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 28 Feb 2022 08:14:39 -0800 Subject: [PATCH 119/551] admin: SHOW CONFIG (#50) * admin: SHOW CONFIG * test --- .circleci/run_tests.sh | 5 +-- src/admin.rs | 91 ++++++++++++++++++++++++++++++++++++++++++ src/config.rs | 46 +++++++++++++++++++++ 3 files changed, 139 insertions(+), 3 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index abc1a7f57..e3119046d 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -59,6 +59,8 @@ cd ../.. # Admin tests psql -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/null +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW CONFIG' > /dev/null (! psql -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) # Start PgCat in debug to demonstrate failover better @@ -86,9 +88,6 @@ sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' pgcat.toml # Reload config test kill -SIGHUP $(pgrep pgcat) -# Reload again with the admin database -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null - # Prepared statements that will only work in session mode pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol prepared diff --git a/src/admin.rs b/src/admin.rs index 97346ed87..43c46afd4 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -2,6 +2,8 @@ use bytes::{Buf, BufMut, BytesMut}; use log::{info, trace}; use tokio::net::tcp::OwnedWriteHalf; +use std::collections::HashMap; + use crate::config::{get_config, parse}; use crate::constants::{OID_NUMERIC, OID_TEXT}; use crate::errors::Error; @@ -27,6 +29,9 @@ pub async fn handle_admin(stream: &mut OwnedWriteHalf, mut query: BytesMut) -> R } else if query.starts_with("RELOAD") { trace!("RELOAD"); reload(stream).await + } else if query.starts_with("SHOW CONFIG") { + trace!("SHOW CONFIG"); + show_config(stream).await } else { Err(Error::ProtocolSyncError) } @@ -61,6 +66,92 @@ pub async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { write_all_half(stream, res).await } +pub async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { + let guard = get_config(); + let config = &*guard.clone(); + let config: HashMap = config.into(); + drop(guard); + + // Configs that cannot be changed dynamically. + let immutables = ["host", "port", "connect_timeout"]; + + // Columns + let columns = ["key", "value", "default", "changeable"]; + + // RowDescription + let mut row_desc = BytesMut::new(); + row_desc.put_i16(4 as i16); // key, value, default, changeable + + for column in columns { + row_desc.put_slice(&format!("{}\0", column).as_bytes()); + + // Doesn't belong to any table + row_desc.put_i32(0); + + // Doesn't belong to any table + row_desc.put_i16(0); + + // Data type + row_desc.put_i32(OID_TEXT); + + // text size = variable (-1) + row_desc.put_i16(-1); + + // Type modifier: none that I know + row_desc.put_i32(-1); + + // Format being used: text (0), binary (1) + row_desc.put_i16(0); + } + + // Response data + let mut res = BytesMut::new(); + res.put_u8(b'T'); + res.put_i32(row_desc.len() as i32 + 4); + res.put(row_desc); + + // DataRow rows + for (key, value) in config { + let mut data_row = BytesMut::new(); + + data_row.put_i16(4 as i16); // key, value, default, changeable + + let key_bytes = key.as_bytes(); + let value = value.as_bytes(); + + data_row.put_i32(key_bytes.len() as i32); + data_row.put_slice(&key_bytes); + + data_row.put_i32(value.len() as i32); + data_row.put_slice(&value); + + data_row.put_i32(1 as i32); + data_row.put_slice(&"-".as_bytes()); + + let changeable = if immutables.iter().filter(|col| *col == &key).count() == 1 { + "no".as_bytes() + } else { + "yes".as_bytes() + }; + data_row.put_i32(changeable.len() as i32); + data_row.put_slice(&changeable); + + res.put_u8(b'D'); + res.put_i32(data_row.len() as i32 + 4); + res.put(data_row); + } + + res.put_u8(b'C'); + res.put_i32("SHOW CONFIG\0".as_bytes().len() as i32 + 4); + res.put_slice(&"SHOW CONFIG\0".as_bytes()); + + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} + /// SHOW STATS pub async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let columns = [ diff --git a/src/config.rs b/src/config.rs index 3cb413a88..203f41021 100644 --- a/src/config.rs +++ b/src/config.rs @@ -153,6 +153,52 @@ impl Default for Config { } } +impl From<&Config> for std::collections::HashMap { + fn from(config: &Config) -> HashMap { + HashMap::from([ + ("host".to_string(), config.general.host.to_string()), + ("port".to_string(), config.general.port.to_string()), + ( + "pool_size".to_string(), + config.general.pool_size.to_string(), + ), + ( + "pool_mode".to_string(), + config.general.pool_mode.to_string(), + ), + ( + "connect_timeout".to_string(), + config.general.connect_timeout.to_string(), + ), + ( + "healthcheck_timeout".to_string(), + config.general.healthcheck_timeout.to_string(), + ), + ("ban_time".to_string(), config.general.ban_time.to_string()), + ( + "statsd_address".to_string(), + config.general.statsd_address.to_string(), + ), + ( + "default_role".to_string(), + config.query_router.default_role.to_string(), + ), + ( + "query_parser_enabled".to_string(), + config.query_router.query_parser_enabled.to_string(), + ), + ( + "primary_reads_enabled".to_string(), + config.query_router.primary_reads_enabled.to_string(), + ), + ( + "sharding_function".to_string(), + config.query_router.sharding_function.to_string(), + ), + ]) + } +} + impl Config { pub fn show(&self) { info!("Pool size: {}", self.general.pool_size); From 6d01d813f1790f1b1e04157dbb60b363470935de Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 28 Feb 2022 17:22:28 -0800 Subject: [PATCH 120/551] admin SHOW DATABASES (#51) * admin SHOW DATABASES * test * correct replica count --- .circleci/run_tests.sh | 1 + src/admin.rs | 157 ++++++++++++++++++++++++++++++++++++++--- src/client.rs | 2 +- src/config.rs | 9 +++ src/constants.rs | 2 + src/pool.rs | 8 +++ 6 files changed, 170 insertions(+), 9 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index e3119046d..22ad483ad 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -61,6 +61,7 @@ cd ../.. psql -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/null psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW CONFIG' > /dev/null +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' > /dev/null (! psql -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) # Start PgCat in debug to demonstrate failover better diff --git a/src/admin.rs b/src/admin.rs index 43c46afd4..fb4a71802 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -4,14 +4,19 @@ use tokio::net::tcp::OwnedWriteHalf; use std::collections::HashMap; -use crate::config::{get_config, parse}; -use crate::constants::{OID_NUMERIC, OID_TEXT}; +use crate::config::{get_config, parse, Role}; +use crate::constants::{OID_INT4, OID_NUMERIC, OID_TEXT}; use crate::errors::Error; -use crate::messages::write_all_half; +use crate::messages::{custom_protocol_response_ok, error_response, write_all_half}; +use crate::pool::ConnectionPool; use crate::stats::get_stats; /// Handle admin client -pub async fn handle_admin(stream: &mut OwnedWriteHalf, mut query: BytesMut) -> Result<(), Error> { +pub async fn handle_admin( + stream: &mut OwnedWriteHalf, + mut query: BytesMut, + pool: ConnectionPool, +) -> Result<(), Error> { let code = query.get_u8() as char; if code != 'Q' { @@ -23,6 +28,8 @@ pub async fn handle_admin(stream: &mut OwnedWriteHalf, mut query: BytesMut) -> R .to_string() .to_ascii_uppercase(); + trace!("Admin query: {}", query); + if query.starts_with("SHOW STATS") { trace!("SHOW STATS"); show_stats(stream).await @@ -32,13 +39,147 @@ pub async fn handle_admin(stream: &mut OwnedWriteHalf, mut query: BytesMut) -> R } else if query.starts_with("SHOW CONFIG") { trace!("SHOW CONFIG"); show_config(stream).await + } else if query.starts_with("SHOW DATABASES") { + trace!("SHOW DATABASES"); + show_databases(stream, &pool).await + } else if query.starts_with("SET ") { + trace!("SET"); + ignore_set(stream).await } else { - Err(Error::ProtocolSyncError) + error_response(stream, "Unsupported query against the admin database").await } } +/// SHOW DATABASES +async fn show_databases(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { + let guard = get_config(); + let config = &*guard.clone(); + drop(guard); + + let columns = [ + "name", + "host", + "port", + "database", + "force_user", + "pool_size", + "min_pool_size", + "reserve_pool", + "pool_mode", + "max_connections", + "current_connections", + "paused", + "disabled", + ]; + + let types = [ + OID_TEXT, OID_TEXT, OID_TEXT, OID_TEXT, OID_TEXT, OID_INT4, OID_INT4, OID_INT4, OID_TEXT, + OID_INT4, OID_INT4, OID_INT4, OID_INT4, + ]; + + let mut res = BytesMut::new(); + let mut row_desc = BytesMut::new(); + row_desc.put_i16(columns.len() as i16); + + for (i, column) in columns.iter().enumerate() { + row_desc.put_slice(&format!("{}\0", column).as_bytes()); + + // Doesn't belong to any table + row_desc.put_i32(0); + + // Doesn't belong to any table + row_desc.put_i16(0); + + // Data type + row_desc.put_i32(types[i]); + + // text size = variable (-1) + row_desc.put_i16(if types[i] == OID_TEXT { -1 } else { 4 }); + + // Type modifier: none that I know + row_desc.put_i32(-1); + + // Format being used: text (0), binary (1) + row_desc.put_i16(0); + } + + res.put_u8(b'T'); + res.put_i32(row_desc.len() as i32 + 4); + res.put(row_desc); + + for shard in 0..pool.shards() { + let database_name = &config.shards[&shard.to_string()].database; + let mut replica_count = 0; + for server in 0..pool.servers(shard) { + // DataRow + let mut data_row = BytesMut::new(); + data_row.put_i16(columns.len() as i16); + + let address = pool.address(shard, server); + let role = address.role.to_string(); + let name = match role.as_ref() { + "primary" => format!("shard_{}_primary", shard), + "replica" => format!("shard_{}_replica_{}", shard, replica_count), + _ => unreachable!(), + }; + let connections = pool.connections(shard, server); + + let data = HashMap::from([ + ("host", address.host.to_string()), + ("port", address.port.to_string()), + ("role", role), + ("name", name), + ("database", database_name.to_string()), + ("force_user", config.user.name.to_string()), + ("pool_size", config.general.pool_size.to_string()), + ("min_pool_size", "0".to_string()), + ("reserve_pool", "0".to_string()), + ("pool_mode", config.general.pool_mode.to_string()), + // There is only one user support at the moment, + // so max_connections = num of users * pool_size = 1 * pool_size. + ("max_connections", config.general.pool_size.to_string()), + ("current_connections", connections.connections.to_string()), + ("paused", "0".to_string()), + ("disabled", "0".to_string()), + ]); + + for column in &columns { + let value = data[column].as_bytes(); + + data_row.put_i32(value.len() as i32); + data_row.put_slice(&value); + } + + res.put_u8(b'D'); + res.put_i32(data_row.len() as i32 + 4); + res.put(data_row); + + if address.role == Role::Replica { + replica_count += 1; + } + } + } + + let command_complete = BytesMut::from(&"SHOW\0"[..]); + res.put_u8(b'C'); + res.put_i32(command_complete.len() as i32 + 4); + res.put(command_complete); + + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} + +/// Ignore any SET commands the client sends. +/// This is common initialization done by ORMs. +async fn ignore_set(stream: &mut OwnedWriteHalf) -> Result<(), Error> { + custom_protocol_response_ok(stream, "SET").await +} + /// RELOAD -pub async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { +async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { info!("Reloading config"); let config = get_config(); @@ -66,7 +207,7 @@ pub async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { write_all_half(stream, res).await } -pub async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { +async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let guard = get_config(); let config = &*guard.clone(); let config: HashMap = config.into(); @@ -153,7 +294,7 @@ pub async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { } /// SHOW STATS -pub async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { +async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let columns = [ "database", "total_xact_count", diff --git a/src/client.rs b/src/client.rs index fb1a4b0ec..f73ef2239 100644 --- a/src/client.rs +++ b/src/client.rs @@ -238,7 +238,7 @@ impl Client { // Handle admin database real quick if self.admin { trace!("Handling admin command"); - handle_admin(&mut self.write, message).await?; + handle_admin(&mut self.write, message, pool.clone()).await?; continue; } diff --git a/src/config.rs b/src/config.rs index 203f41021..1687ff310 100644 --- a/src/config.rs +++ b/src/config.rs @@ -19,6 +19,15 @@ pub enum Role { Replica, } +impl ToString for Role { + fn to_string(&self) -> String { + match *self { + Role::Primary => "primary".to_string(), + Role::Replica => "replica".to_string(), + } + } +} + impl PartialEq> for Role { fn eq(&self, other: &Option) -> bool { match other { diff --git a/src/constants.rs b/src/constants.rs index 25609759a..b829eaa01 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -26,3 +26,5 @@ pub const MESSAGE_TERMINATOR: u8 = 0; // pub const OID_NUMERIC: i32 = 1700; pub const OID_TEXT: i32 = 25; +pub const OID_INT4: i32 = 23; // int +pub const _OID_INT8: i32 = 20; // bigint diff --git a/src/pool.rs b/src/pool.rs index 26d172bea..431a909a8 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -336,6 +336,14 @@ impl ConnectionPool { pub fn servers(&self, shard: usize) -> usize { self.addresses[shard].len() } + + pub fn connections(&self, shard: usize, server: usize) -> bb8::State { + self.databases[shard][server].state() + } + + pub fn address(&self, shard: usize, server: usize) -> &Address { + &self.addresses[shard][server] + } } pub struct ServerPool { From cf3117fdf4bd8c078a5a640fb2a149a67771a130 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 1 Mar 2022 08:47:19 -0800 Subject: [PATCH 121/551] Refactor admin (#52) --- src/admin.rs | 307 +++++++++++++---------------------------------- src/constants.rs | 3 - src/messages.rs | 127 +++++++++++++------- src/pool.rs | 2 +- 4 files changed, 167 insertions(+), 272 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index fb4a71802..0d2e19740 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -5,9 +5,8 @@ use tokio::net::tcp::OwnedWriteHalf; use std::collections::HashMap; use crate::config::{get_config, parse, Role}; -use crate::constants::{OID_INT4, OID_NUMERIC, OID_TEXT}; use crate::errors::Error; -use crate::messages::{custom_protocol_response_ok, error_response, write_all_half}; +use crate::messages::*; use crate::pool::ConnectionPool; use crate::stats::get_stats; @@ -56,115 +55,66 @@ async fn show_databases(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> R let config = &*guard.clone(); drop(guard); - let columns = [ - "name", - "host", - "port", - "database", - "force_user", - "pool_size", - "min_pool_size", - "reserve_pool", - "pool_mode", - "max_connections", - "current_connections", - "paused", - "disabled", - ]; - - let types = [ - OID_TEXT, OID_TEXT, OID_TEXT, OID_TEXT, OID_TEXT, OID_INT4, OID_INT4, OID_INT4, OID_TEXT, - OID_INT4, OID_INT4, OID_INT4, OID_INT4, + // Columns + let columns = vec![ + ("name", DataType::Text), + ("host", DataType::Text), + ("port", DataType::Text), + ("database", DataType::Text), + ("force_user", DataType::Text), + ("pool_size", DataType::Int4), + ("min_pool_size", DataType::Int4), + ("reserve_pool", DataType::Int4), + ("pool_mode", DataType::Text), + ("max_connections", DataType::Int4), + ("current_connections", DataType::Int4), + ("paused", DataType::Int4), + ("disabled", DataType::Int4), ]; let mut res = BytesMut::new(); - let mut row_desc = BytesMut::new(); - row_desc.put_i16(columns.len() as i16); - - for (i, column) in columns.iter().enumerate() { - row_desc.put_slice(&format!("{}\0", column).as_bytes()); - - // Doesn't belong to any table - row_desc.put_i32(0); - - // Doesn't belong to any table - row_desc.put_i16(0); - // Data type - row_desc.put_i32(types[i]); - - // text size = variable (-1) - row_desc.put_i16(if types[i] == OID_TEXT { -1 } else { 4 }); - - // Type modifier: none that I know - row_desc.put_i32(-1); - - // Format being used: text (0), binary (1) - row_desc.put_i16(0); - } - - res.put_u8(b'T'); - res.put_i32(row_desc.len() as i32 + 4); - res.put(row_desc); + // RowDescription + res.put(row_description(&columns)); for shard in 0..pool.shards() { let database_name = &config.shards[&shard.to_string()].database; let mut replica_count = 0; - for server in 0..pool.servers(shard) { - // DataRow - let mut data_row = BytesMut::new(); - data_row.put_i16(columns.len() as i16); + for server in 0..pool.servers(shard) { let address = pool.address(shard, server); - let role = address.role.to_string(); - let name = match role.as_ref() { - "primary" => format!("shard_{}_primary", shard), - "replica" => format!("shard_{}_replica_{}", shard, replica_count), - _ => unreachable!(), + let name = match address.role { + Role::Primary => format!("shard_{}_primary", shard), + + Role::Replica => { + let name = format!("shard_{}_replica_{}", shard, replica_count); + replica_count += 1; + name + } }; - let connections = pool.connections(shard, server); - - let data = HashMap::from([ - ("host", address.host.to_string()), - ("port", address.port.to_string()), - ("role", role), - ("name", name), - ("database", database_name.to_string()), - ("force_user", config.user.name.to_string()), - ("pool_size", config.general.pool_size.to_string()), - ("min_pool_size", "0".to_string()), - ("reserve_pool", "0".to_string()), - ("pool_mode", config.general.pool_mode.to_string()), - // There is only one user support at the moment, - // so max_connections = num of users * pool_size = 1 * pool_size. - ("max_connections", config.general.pool_size.to_string()), - ("current_connections", connections.connections.to_string()), - ("paused", "0".to_string()), - ("disabled", "0".to_string()), - ]); - - for column in &columns { - let value = data[column].as_bytes(); - - data_row.put_i32(value.len() as i32); - data_row.put_slice(&value); - } - - res.put_u8(b'D'); - res.put_i32(data_row.len() as i32 + 4); - res.put(data_row); - - if address.role == Role::Replica { - replica_count += 1; - } + let pool_state = pool.pool_state(shard, server); + + res.put(data_row(&vec![ + name, // name + address.host.to_string(), // host + address.port.to_string(), // port + database_name.to_string(), // database + config.user.name.to_string(), // force_user + config.general.pool_size.to_string(), // pool_size + "0".to_string(), // min_pool_size + "0".to_string(), // reserve_pool + config.general.pool_mode.to_string(), // pool_mode + config.general.pool_size.to_string(), // max_connections + pool_state.connections.to_string(), // current_connections + "0".to_string(), // paused + "0".to_string(), // disabled + ])); } } - let command_complete = BytesMut::from(&"SHOW\0"[..]); - res.put_u8(b'C'); - res.put_i32(command_complete.len() as i32 + 4); - res.put(command_complete); + res.put(command_complete("SHOW")); + // ReadyForQuery res.put_u8(b'Z'); res.put_i32(5); res.put_u8(b'I'); @@ -194,10 +144,7 @@ async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let mut res = BytesMut::new(); // CommandComplete - let command_complete = BytesMut::from(&"RELOAD\0"[..]); - res.put_u8(b'C'); - res.put_i32(command_complete.len() as i32 + 4); - res.put(command_complete); + res.put(command_complete("RELOAD")); // ReadyForQuery res.put_u8(b'Z'); @@ -217,74 +164,31 @@ async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let immutables = ["host", "port", "connect_timeout"]; // Columns - let columns = ["key", "value", "default", "changeable"]; - - // RowDescription - let mut row_desc = BytesMut::new(); - row_desc.put_i16(4 as i16); // key, value, default, changeable - - for column in columns { - row_desc.put_slice(&format!("{}\0", column).as_bytes()); - - // Doesn't belong to any table - row_desc.put_i32(0); - - // Doesn't belong to any table - row_desc.put_i16(0); - - // Data type - row_desc.put_i32(OID_TEXT); - - // text size = variable (-1) - row_desc.put_i16(-1); - - // Type modifier: none that I know - row_desc.put_i32(-1); - - // Format being used: text (0), binary (1) - row_desc.put_i16(0); - } + let columns = vec![ + ("key", DataType::Text), + ("value", DataType::Text), + ("default", DataType::Text), + ("changeable", DataType::Text), + ]; // Response data let mut res = BytesMut::new(); - res.put_u8(b'T'); - res.put_i32(row_desc.len() as i32 + 4); - res.put(row_desc); + res.put(row_description(&columns)); // DataRow rows for (key, value) in config { - let mut data_row = BytesMut::new(); - - data_row.put_i16(4 as i16); // key, value, default, changeable - - let key_bytes = key.as_bytes(); - let value = value.as_bytes(); - - data_row.put_i32(key_bytes.len() as i32); - data_row.put_slice(&key_bytes); - - data_row.put_i32(value.len() as i32); - data_row.put_slice(&value); - - data_row.put_i32(1 as i32); - data_row.put_slice(&"-".as_bytes()); - let changeable = if immutables.iter().filter(|col| *col == &key).count() == 1 { - "no".as_bytes() + "no".to_string() } else { - "yes".as_bytes() + "yes".to_string() }; - data_row.put_i32(changeable.len() as i32); - data_row.put_slice(&changeable); - res.put_u8(b'D'); - res.put_i32(data_row.len() as i32 + 4); - res.put(data_row); + let row = vec![key, value, "-".to_string(), changeable]; + + res.put(data_row(&row)); } - res.put_u8(b'C'); - res.put_i32("SHOW CONFIG\0".as_bytes().len() as i32 + 4); - res.put_slice(&"SHOW CONFIG\0".as_bytes()); + res.put(command_complete("SHOW")); res.put_u8(b'Z'); res.put_i32(5); @@ -295,81 +199,38 @@ async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { /// SHOW STATS async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { - let columns = [ - "database", - "total_xact_count", - "total_query_count", - "total_received", - "total_sent", - "total_xact_time", - "total_query_time", - "total_wait_time", - "avg_xact_count", - "avg_query_count", - "avg_recv", - "avg_sent", - "avg_xact_time", - "avg_query_time", - "avg_wait_time", + let columns = vec![ + ("database", DataType::Text), + ("total_xact_count", DataType::Numeric), + ("total_query_count", DataType::Numeric), + ("total_received", DataType::Numeric), + ("total_sent", DataType::Numeric), + ("total_xact_time", DataType::Numeric), + ("total_query_time", DataType::Numeric), + ("total_wait_time", DataType::Numeric), + ("avg_xact_count", DataType::Numeric), + ("avg_query_count", DataType::Numeric), + ("avg_recv", DataType::Numeric), + ("avg_sent", DataType::Numeric), + ("avg_xact_time", DataType::Numeric), + ("avg_query_time", DataType::Numeric), + ("avg_wait_time", DataType::Numeric), ]; let stats = get_stats(); let mut res = BytesMut::new(); - let mut row_desc = BytesMut::new(); - let mut data_row = BytesMut::new(); - - // Number of columns: 1 - row_desc.put_i16(columns.len() as i16); - data_row.put_i16(columns.len() as i16); - - for (i, column) in columns.iter().enumerate() { - // RowDescription - - // Column name - row_desc.put_slice(&format!("{}\0", column).as_bytes()); + res.put(row_description(&columns)); - // Doesn't belong to any table - row_desc.put_i32(0); - - // Doesn't belong to any table - row_desc.put_i16(0); - - // Data type - row_desc.put_i32(if i == 0 { OID_TEXT } else { OID_NUMERIC }); - - // Numeric/text size = variable (-1) - row_desc.put_i16(-1); - - // Type modifier: none that I know - row_desc.put_i32(-1); - - // Format being used: text (0), binary (1) - row_desc.put_i16(0); - - // DataRow - let value = if i == 0 { - String::from("all shards") - } else { - stats.get(&column.to_string()).unwrap_or(&0).to_string() - }; + let mut row = vec![ + String::from("all shards"), // TODO: per-database stats, + ]; - data_row.put_i32(value.len() as i32); - data_row.put_slice(value.as_bytes()); + for column in &columns[1..] { + row.push(stats.get(column.0).unwrap_or(&0).to_string()); } - let command_complete = BytesMut::from(&"SHOW\0"[..]); - - res.put_u8(b'T'); - res.put_i32(row_desc.len() as i32 + 4); - res.put(row_desc); - - res.put_u8(b'D'); - res.put_i32(data_row.len() as i32 + 4); - res.put(data_row); - - res.put_u8(b'C'); - res.put_i32(command_complete.len() as i32 + 4); - res.put(command_complete); + res.put(data_row(&row)); + res.put(command_complete("SHOW")); res.put_u8(b'Z'); res.put_i32(5); diff --git a/src/constants.rs b/src/constants.rs index b829eaa01..074811e1d 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -24,7 +24,4 @@ pub const MESSAGE_TERMINATOR: u8 = 0; // // Data types // -pub const OID_NUMERIC: i32 = 1700; -pub const OID_TEXT: i32 = 25; -pub const OID_INT4: i32 = 23; // int pub const _OID_INT8: i32 = 20; // bigint diff --git a/src/messages.rs b/src/messages.rs index 8f7080f44..473c8de6b 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -8,9 +8,26 @@ use tokio::net::{ TcpStream, }; +use crate::errors::Error; use std::collections::HashMap; -use crate::errors::Error; +/// Postgres data type mappings +/// used in RowDescription ('T') message. +pub enum DataType { + Text, + Int4, + Numeric, +} + +impl From<&DataType> for i32 { + fn from(data_type: &DataType) -> i32 { + match data_type { + DataType::Text => 25, + DataType::Int4 => 23, + DataType::Numeric => 1700, + } + } +} /// Tell the client that authentication handshake completed successfully. pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { @@ -259,75 +276,95 @@ pub async fn show_response( // 3. CommandComplete // 4. ReadyForQuery - // RowDescription - let mut row_desc = BytesMut::new(); + // The final messages sent to the client + let mut res = BytesMut::new(); - // Number of columns: 1 - row_desc.put_i16(1); + // RowDescription + res.put(row_description(&vec![(name, DataType::Text)])); - // Column name - row_desc.put_slice(&format!("{}\0", name).as_bytes()); + // DataRow + res.put(data_row(&vec![value.to_string()])); - // Doesn't belong to any table - row_desc.put_i32(0); + // CommandComplete + res.put(command_complete("SELECT 1")); - // Doesn't belong to any table - row_desc.put_i16(0); + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); - // Text - row_desc.put_i32(25); + write_all_half(stream, res).await +} - // Text size = variable (-1) - row_desc.put_i16(-1); +pub fn row_description(columns: &Vec<(&str, DataType)>) -> BytesMut { + let mut res = BytesMut::new(); + let mut row_desc = BytesMut::new(); - // Type modifier: none that I know - row_desc.put_i32(-1); + // how many colums we are storing + row_desc.put_i16(columns.len() as i16); - // Format being used: text (0), binary (1) - row_desc.put_i16(0); + for (name, data_type) in columns { + // Column name + row_desc.put_slice(&format!("{}\0", name).as_bytes()); - // DataRow - let mut data_row = BytesMut::new(); + // Doesn't belong to any table + row_desc.put_i32(0); - // Number of columns - data_row.put_i16(1); + // Doesn't belong to any table + row_desc.put_i16(0); - // Size of the column content (length of the string really) - data_row.put_i32(value.len() as i32); + // Text + row_desc.put_i32(data_type.into()); - // The content - data_row.put_slice(value.as_bytes()); + // Text size = variable (-1) + let type_size = match data_type { + DataType::Text => -1, + DataType::Int4 => 4, + DataType::Numeric => -1, + }; - // CommandComplete - let mut command_complete = BytesMut::new(); + row_desc.put_i16(type_size); - // Number of rows returned (just one) - command_complete.put_slice(&b"SELECT 1\0"[..]); + // Type modifier: none that I know + row_desc.put_i32(-1); - // The final messages sent to the client - let mut res = BytesMut::new(); + // Format being used: text (0), binary (1) + row_desc.put_i16(0); + } - // RowDescription res.put_u8(b'T'); res.put_i32(row_desc.len() as i32 + 4); res.put(row_desc); - // DataRow + res +} + +pub fn data_row(row: &Vec) -> BytesMut { + let mut res = BytesMut::new(); + let mut data_row = BytesMut::new(); + + data_row.put_i16(row.len() as i16); + + for column in row { + let column = column.as_bytes(); + data_row.put_i32(column.len() as i32); + data_row.put_slice(&column); + } + res.put_u8(b'D'); res.put_i32(data_row.len() as i32 + 4); res.put(data_row); - // CommandComplete - res.put_u8(b'C'); - res.put_i32(command_complete.len() as i32 + 4); - res.put(command_complete); - - // ReadyForQuery - res.put_u8(b'Z'); - res.put_i32(5); - res.put_u8(b'I'); + res +} - write_all_half(stream, res).await +pub fn command_complete(command: &str) -> BytesMut { + let cmd = BytesMut::from(format!("{}\0", command).as_bytes()); + let mut res = BytesMut::new(); + res.put_u8(b'C'); + res.put_i32(cmd.len() as i32 + 4); + res.put(cmd); + res } /// Write all data in the buffer to the TcpStream. diff --git a/src/pool.rs b/src/pool.rs index 431a909a8..92b76634c 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -337,7 +337,7 @@ impl ConnectionPool { self.addresses[shard].len() } - pub fn connections(&self, shard: usize, server: usize) -> bb8::State { + pub fn pool_state(&self, shard: usize, server: usize) -> bb8::State { self.databases[shard][server].state() } From 91343217ed2e5e89a9cf9af64fd7ddba4afeae5f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 1 Mar 2022 22:49:43 -0800 Subject: [PATCH 122/551] More admin (#53) * more admin * more admin * show lists * tests --- .circleci/run_tests.sh | 4 ++ src/admin.rs | 137 +++++++++++++++++++++++++++++++++++++---- src/config.rs | 12 ++++ src/pool.rs | 20 +++++- 4 files changed, 157 insertions(+), 16 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 22ad483ad..12d19a434 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -62,6 +62,10 @@ psql -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/null psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW CONFIG' > /dev/null psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' > /dev/null +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW LISTS' > /dev/null +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW POOLS' > /dev/null +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW VERSION' > /dev/null +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c "SET client_encoding TO 'utf8'" > /dev/null # will ignore (! psql -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) # Start PgCat in debug to demonstrate failover better diff --git a/src/admin.rs b/src/admin.rs index 0d2e19740..3301c640d 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -4,7 +4,7 @@ use tokio::net::tcp::OwnedWriteHalf; use std::collections::HashMap; -use crate::config::{get_config, parse, Role}; +use crate::config::{get_config, parse}; use crate::errors::Error; use crate::messages::*; use crate::pool::ConnectionPool; @@ -41,6 +41,15 @@ pub async fn handle_admin( } else if query.starts_with("SHOW DATABASES") { trace!("SHOW DATABASES"); show_databases(stream, &pool).await + } else if query.starts_with("SHOW POOLS") { + trace!("SHOW POOLS"); + show_pools(stream, &pool).await + } else if query.starts_with("SHOW LISTS") { + trace!("SHOW LISTS"); + show_lists(stream, &pool).await + } else if query.starts_with("SHOW VERSION") { + trace!("SHOW VERSION"); + show_version(stream).await } else if query.starts_with("SET ") { trace!("SET"); ignore_set(stream).await @@ -49,6 +58,118 @@ pub async fn handle_admin( } } +/// SHOW LISTS +async fn show_lists(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { + let stats = get_stats(); + + let columns = vec![("list", DataType::Text), ("items", DataType::Int4)]; + + let mut res = BytesMut::new(); + res.put(row_description(&columns)); + res.put(data_row(&vec![ + "databases".to_string(), + pool.databases().to_string(), + ])); + res.put(data_row(&vec!["users".to_string(), "1".to_string()])); + res.put(data_row(&vec![ + "pools".to_string(), + pool.databases().to_string(), + ])); + res.put(data_row(&vec![ + "free_clients".to_string(), + stats["cl_idle"].to_string(), + ])); + res.put(data_row(&vec![ + "used_clients".to_string(), + stats["cl_active"].to_string(), + ])); + res.put(data_row(&vec![ + "login_clients".to_string(), + "0".to_string(), + ])); + res.put(data_row(&vec![ + "free_servers".to_string(), + stats["sv_idle"].to_string(), + ])); + res.put(data_row(&vec![ + "used_servers".to_string(), + stats["sv_active"].to_string(), + ])); + res.put(data_row(&vec!["dns_names".to_string(), "0".to_string()])); + res.put(data_row(&vec!["dns_zones".to_string(), "0".to_string()])); + res.put(data_row(&vec!["dns_queries".to_string(), "0".to_string()])); + res.put(data_row(&vec!["dns_pending".to_string(), "0".to_string()])); + + res.put(command_complete("SHOW")); + + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} + +/// SHOW VERSION +async fn show_version(stream: &mut OwnedWriteHalf) -> Result<(), Error> { + let mut res = BytesMut::new(); + + res.put(row_description(&vec![("version", DataType::Text)])); + res.put(data_row(&vec!["PgCat 0.1.0".to_string()])); + res.put(command_complete("SHOW")); + + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} + +/// SHOW POOLS +async fn show_pools(stream: &mut OwnedWriteHalf, _pool: &ConnectionPool) -> Result<(), Error> { + let stats = get_stats(); + let config = { + let guard = get_config(); + &*guard.clone() + }; + + let columns = vec![ + ("database", DataType::Text), + ("user", DataType::Text), + ("cl_active", DataType::Numeric), + ("cl_waiting", DataType::Numeric), + ("cl_cancel_req", DataType::Numeric), + ("sv_active", DataType::Numeric), + ("sv_idle", DataType::Numeric), + ("sv_used", DataType::Numeric), + ("sv_tested", DataType::Numeric), + ("sv_login", DataType::Numeric), + ("maxwait", DataType::Numeric), + ("maxwait_us", DataType::Numeric), + ("pool_mode", DataType::Text), + ]; + + let mut res = BytesMut::new(); + res.put(row_description(&columns)); + + let mut row = vec![String::from("all"), config.user.name.clone()]; + + for column in &columns[2..columns.len() - 1] { + let value = stats.get(column.0).unwrap_or(&0).to_string(); + row.push(value); + } + + row.push(config.general.pool_mode.to_string()); + + res.put(data_row(&row)); + res.put(command_complete("SHOW")); + + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} + /// SHOW DATABASES async fn show_databases(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { let guard = get_config(); @@ -79,23 +200,13 @@ async fn show_databases(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> R for shard in 0..pool.shards() { let database_name = &config.shards[&shard.to_string()].database; - let mut replica_count = 0; for server in 0..pool.servers(shard) { let address = pool.address(shard, server); - let name = match address.role { - Role::Primary => format!("shard_{}_primary", shard), - - Role::Replica => { - let name = format!("shard_{}_replica_{}", shard, replica_count); - replica_count += 1; - name - } - }; let pool_state = pool.pool_state(shard, server); res.put(data_row(&vec![ - name, // name + address.name(), // name address.host.to_string(), // host address.port.to_string(), // port database_name.to_string(), // database @@ -222,7 +333,7 @@ async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { res.put(row_description(&columns)); let mut row = vec![ - String::from("all shards"), // TODO: per-database stats, + String::from("all"), // TODO: per-database stats, ]; for column in &columns[1..] { diff --git a/src/config.rs b/src/config.rs index 1687ff310..663a9f429 100644 --- a/src/config.rs +++ b/src/config.rs @@ -52,6 +52,7 @@ pub struct Address { pub port: String, pub shard: usize, pub role: Role, + pub replica_number: usize, } impl Default for Address { @@ -60,11 +61,22 @@ impl Default for Address { host: String::from("127.0.0.1"), port: String::from("5432"), shard: 0, + replica_number: 0, role: Role::Replica, } } } +impl Address { + pub fn name(&self) -> String { + match self.role { + Role::Primary => format!("shard_{}_primary", self.shard), + + Role::Replica => format!("shard_{}_replica_{}", self.shard, self.replica_number), + } + } +} + #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Deserialize, Debug)] pub struct User { pub name: String, diff --git a/src/pool.rs b/src/pool.rs index 92b76634c..b8a0ecf1f 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -49,7 +49,8 @@ impl ConnectionPool { for shard_idx in shard_ids { let shard = &config.shards[&shard_idx]; let mut pools = Vec::new(); - let mut replica_addresses = Vec::new(); + let mut servers = Vec::new(); + let mut replica_number = 0; for server in shard.servers.iter() { let role = match server.2.as_ref() { @@ -65,9 +66,14 @@ impl ConnectionPool { host: server.0.clone(), port: server.1.to_string(), role: role, + replica_number, shard: shard_idx.parse::().unwrap(), }; + if role == Role::Replica { + replica_number += 1; + } + let manager = ServerPool::new( address.clone(), config.user.clone(), @@ -87,11 +93,11 @@ impl ConnectionPool { .unwrap(); pools.push(pool); - replica_addresses.push(address); + servers.push(address); } shards.push(pools); - addresses.push(replica_addresses); + addresses.push(servers); banlist.push(HashMap::new()); } @@ -337,6 +343,14 @@ impl ConnectionPool { self.addresses[shard].len() } + pub fn databases(&self) -> usize { + let mut databases = 0; + for shard in 0..self.shards() { + databases += self.servers(shard); + } + databases + } + pub fn pool_state(&self, shard: usize, server: usize) -> bb8::State { self.databases[shard][server].state() } From 8c850f31c24249ec198c2a090000838f1da231db Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 2 Mar 2022 14:46:31 -0800 Subject: [PATCH 123/551] Fix pgbouncerhero (#54) --- src/admin.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 3301c640d..39fb3bebd 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -68,13 +68,13 @@ async fn show_lists(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul res.put(row_description(&columns)); res.put(data_row(&vec![ "databases".to_string(), - pool.databases().to_string(), + (pool.databases() + 1).to_string(), // see comment below ])); res.put(data_row(&vec!["users".to_string(), "1".to_string()])); res.put(data_row(&vec![ "pools".to_string(), - pool.databases().to_string(), - ])); + (pool.databases() + 1).to_string(), // +1 for the pgbouncer admin db pool which isn't real + ])); // but admin tools that work with pgbouncer want this res.put(data_row(&vec![ "free_clients".to_string(), stats["cl_idle"].to_string(), From 083affa8e996020a1604bcc3b67964f622929123 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 4 Mar 2022 17:04:27 -0800 Subject: [PATCH 124/551] Per-shard statistics (#57) * per shard stats * aight * cleaner * fix show lists * comments * more friendly * case-insensitive * test all shards * ok * HUH? --- .circleci/run_tests.sh | 9 +- Cargo.lock | 10 --- Cargo.toml | 1 - src/admin.rs | 75 ++++++++++++----- src/client.rs | 62 +++++++++----- src/config.rs | 2 + src/main.rs | 13 +-- src/pool.rs | 40 +++++++-- src/query_router.rs | 7 +- src/server.rs | 7 +- src/stats.rs | 172 +++++++++++++++++++++++---------------- tests/pgbench/simple.sql | 28 +++++++ 12 files changed, 285 insertions(+), 141 deletions(-) create mode 100644 tests/pgbench/simple.sql diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 12d19a434..9dbca9d20 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -13,6 +13,9 @@ function start_pgcat() { # Setup the database with shards and user psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql +PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard0 -i +PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i +PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard2 -i # Install Toxiproxy to simulate a downed/slow database wget -O toxiproxy-2.1.4.deb https://github.com/Shopify/toxiproxy/releases/download/v2.1.4/toxiproxy_2.1.4_amd64.deb @@ -28,9 +31,9 @@ toxiproxy-cli create -l 127.0.0.1:5433 -u 127.0.0.1:5432 postgres_replica start_pgcat "info" # pgbench test -pgbench -i -h 127.0.0.1 -p 6432 && \ - pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol simple && \ - pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended +pgbench -i -h 127.0.0.1 -p 6432 +pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol simple -f tests/pgbench/simple.sql +pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended # COPY TO STDOUT test psql -h 127.0.0.1 -p 6432 -c 'COPY (SELECT * FROM pgbench_accounts LIMIT 15) TO STDOUT;' > /dev/null diff --git a/Cargo.lock b/Cargo.lock index d6e42cb7e..7ee21380a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,7 +371,6 @@ dependencies = [ "serde_derive", "sha-1", "sqlparser", - "statsd", "tokio", "toml", ] @@ -542,15 +541,6 @@ dependencies = [ "log", ] -[[package]] -name = "statsd" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1efceb4bf2c0b5ebec94354285a43bbbed1375605bdf2ebe4132299434a330" -dependencies = [ - "rand", -] - [[package]] name = "syn" version = "1.0.86" diff --git a/Cargo.toml b/Cargo.toml index d070c61da..74d4a76a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,6 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -statsd = "0.15" sqlparser = "0.14" log = "0.4" arc-swap = "1" diff --git a/src/admin.rs b/src/admin.rs index 39fb3bebd..8f94b5db1 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -31,7 +31,7 @@ pub async fn handle_admin( if query.starts_with("SHOW STATS") { trace!("SHOW STATS"); - show_stats(stream).await + show_stats(stream, &pool).await } else if query.starts_with("RELOAD") { trace!("RELOAD"); reload(stream).await @@ -77,11 +77,19 @@ async fn show_lists(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul ])); // but admin tools that work with pgbouncer want this res.put(data_row(&vec![ "free_clients".to_string(), - stats["cl_idle"].to_string(), + stats + .keys() + .map(|address_id| stats[&address_id]["cl_idle"]) + .sum::() + .to_string(), ])); res.put(data_row(&vec![ "used_clients".to_string(), - stats["cl_active"].to_string(), + stats + .keys() + .map(|address_id| stats[&address_id]["cl_active"]) + .sum::() + .to_string(), ])); res.put(data_row(&vec![ "login_clients".to_string(), @@ -89,11 +97,19 @@ async fn show_lists(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul ])); res.put(data_row(&vec![ "free_servers".to_string(), - stats["sv_idle"].to_string(), + stats + .keys() + .map(|address_id| stats[&address_id]["sv_idle"]) + .sum::() + .to_string(), ])); res.put(data_row(&vec![ "used_servers".to_string(), - stats["sv_active"].to_string(), + stats + .keys() + .map(|address_id| stats[&address_id]["sv_active"]) + .sum::() + .to_string(), ])); res.put(data_row(&vec!["dns_names".to_string(), "0".to_string()])); res.put(data_row(&vec!["dns_zones".to_string(), "0".to_string()])); @@ -125,7 +141,7 @@ async fn show_version(stream: &mut OwnedWriteHalf) -> Result<(), Error> { } /// SHOW POOLS -async fn show_pools(stream: &mut OwnedWriteHalf, _pool: &ConnectionPool) -> Result<(), Error> { +async fn show_pools(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { let stats = get_stats(); let config = { let guard = get_config(); @@ -151,16 +167,26 @@ async fn show_pools(stream: &mut OwnedWriteHalf, _pool: &ConnectionPool) -> Resu let mut res = BytesMut::new(); res.put(row_description(&columns)); - let mut row = vec![String::from("all"), config.user.name.clone()]; + for shard in 0..pool.shards() { + for server in 0..pool.servers(shard) { + let address = pool.address(shard, server); + let stats = match stats.get(&address.id) { + Some(stats) => stats.clone(), + None => HashMap::new(), + }; - for column in &columns[2..columns.len() - 1] { - let value = stats.get(column.0).unwrap_or(&0).to_string(); - row.push(value); - } + let mut row = vec![address.name(), config.user.name.clone()]; - row.push(config.general.pool_mode.to_string()); + for column in &columns[2..columns.len() - 1] { + let value = stats.get(column.0).unwrap_or(&0).to_string(); + row.push(value); + } + + row.push(config.general.pool_mode.to_string()); + res.put(data_row(&row)); + } + } - res.put(data_row(&row)); res.put(command_complete("SHOW")); res.put_u8(b'Z'); @@ -309,7 +335,7 @@ async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { } /// SHOW STATS -async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { +async fn show_stats(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { let columns = vec![ ("database", DataType::Text), ("total_xact_count", DataType::Numeric), @@ -332,15 +358,24 @@ async fn show_stats(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let mut res = BytesMut::new(); res.put(row_description(&columns)); - let mut row = vec![ - String::from("all"), // TODO: per-database stats, - ]; + for shard in 0..pool.shards() { + for server in 0..pool.servers(shard) { + let address = pool.address(shard, server); + let stats = match stats.get(&address.id) { + Some(stats) => stats.clone(), + None => HashMap::new(), + }; - for column in &columns[1..] { - row.push(stats.get(column.0).unwrap_or(&0).to_string()); + let mut row = vec![address.name()]; + + for column in &columns[1..] { + row.push(stats.get(column.0).unwrap_or(&0).to_string()); + } + + res.put(data_row(&row)); + } } - res.put(data_row(&row)); res.put(command_complete("SHOW")); res.put_u8(b'Z'); diff --git a/src/client.rs b/src/client.rs index f73ef2239..0fc9c4749 100644 --- a/src/client.rs +++ b/src/client.rs @@ -58,6 +58,12 @@ pub struct Client { // Clients want to talk to admin admin: bool, + + // Last address the client talked to + last_address_id: Option, + + // Last server process id we talked to + last_server_id: Option, } impl Client { @@ -147,6 +153,8 @@ impl Client { parameters: parameters, stats: stats, admin: admin, + last_address_id: None, + last_server_id: None, }); } @@ -169,6 +177,8 @@ impl Client { parameters: HashMap::new(), stats: stats, admin: false, + last_address_id: None, + last_server_id: None, }); } @@ -219,9 +229,6 @@ impl Client { loop { trace!("Client idle, waiting for message"); - // Client idle, waiting for messages. - self.stats.client_idle(self.process_id); - // Read a complete message from the client, which normally would be // either a `Q` (query) or `P` (prepare, extended protocol). // We can parse it here before grabbing a server from the pool, @@ -292,13 +299,13 @@ impl Client { continue; } - // Waiting for server connection. - self.stats.client_waiting(self.process_id); - debug!("Waiting for connection from pool"); // Grab a server from the pool: the client issued a regular query. - let connection = match pool.get(query_router.shard(), query_router.role()).await { + let connection = match pool + .get(query_router.shard(), query_router.role(), self.process_id) + .await + { Ok(conn) => { debug!("Got connection from pool"); conn @@ -312,15 +319,23 @@ impl Client { }; let mut reference = connection.0; - let _address = connection.1; + let address = connection.1; let server = &mut *reference; // Claim this server as mine for query cancellation. server.claim(self.process_id, self.secret_key); + // "disconnect" from the previous server stats-wise + if let Some(last_address_id) = self.last_address_id { + self.stats + .client_disconnecting(self.process_id, last_address_id); + } + // Client active & server active - self.stats.client_active(self.process_id); - self.stats.server_active(server.process_id()); + self.stats.client_active(self.process_id, address.id); + self.stats.server_active(server.process_id(), address.id); + self.last_address_id = Some(address.id); + self.last_server_id = Some(server.process_id()); debug!( "Client {:?} talking to server {:?}", @@ -392,17 +407,17 @@ impl Client { } // Report query executed statistics. - self.stats.query(); + self.stats.query(address.id); // The transaction is over, we can release the connection back to the pool. if !server.in_transaction() { // Report transaction executed statistics. - self.stats.transaction(); + self.stats.transaction(address.id); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { - self.stats.server_idle(server.process_id()); + self.stats.server_idle(server.process_id(), address.id); break; } } @@ -478,15 +493,15 @@ impl Client { } // Report query executed statistics. - self.stats.query(); + self.stats.query(address.id); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if !server.in_transaction() { - self.stats.transaction(); + self.stats.transaction(address.id); if self.transaction_mode { - self.stats.server_idle(server.process_id()); + self.stats.server_idle(server.process_id(), address.id); break; } } @@ -517,10 +532,10 @@ impl Client { // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if !server.in_transaction() { - self.stats.transaction(); + self.stats.transaction(address.id); if self.transaction_mode { - self.stats.server_idle(server.process_id()); + self.stats.server_idle(server.process_id(), address.id); break; } } @@ -537,6 +552,7 @@ impl Client { // The server is no longer bound to us, we can't cancel it's queries anymore. debug!("Releasing server back into the pool"); self.release(); + self.stats.client_idle(self.process_id, address.id); } } @@ -549,6 +565,14 @@ impl Client { impl Drop for Client { fn drop(&mut self) { - self.stats.client_disconnecting(self.process_id); + // Disconnect the client + if let Some(address_id) = self.last_address_id { + self.stats.client_disconnecting(self.process_id, address_id); + + // The server is now idle + if let Some(process_id) = self.last_server_id { + self.stats.server_idle(process_id, address_id); + } + } } } diff --git a/src/config.rs b/src/config.rs index 663a9f429..cd8b2eb61 100644 --- a/src/config.rs +++ b/src/config.rs @@ -48,6 +48,7 @@ impl PartialEq for Option { #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)] pub struct Address { + pub id: usize, pub host: String, pub port: String, pub shard: usize, @@ -58,6 +59,7 @@ pub struct Address { impl Default for Address { fn default() -> Address { Address { + id: 0, host: String::from("127.0.0.1"), port: String::from("5432"), shard: 0, diff --git a/src/main.rs b/src/main.rs index a41d5ecc6..ada4551f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,7 +31,6 @@ extern crate once_cell; extern crate serde; extern crate serde_derive; extern crate sqlparser; -extern crate statsd; extern crate tokio; extern crate toml; @@ -113,15 +112,19 @@ async fn main() { // Collect statistics and send them to StatsD let (tx, rx) = mpsc::channel(100); + + // Connection pool for all shards and replicas + let mut pool = + ConnectionPool::from_config(client_server_map.clone(), Reporter::new(tx.clone())).await; + let collector_tx = tx.clone(); + let addresses = pool.databases(); tokio::task::spawn(async move { let mut stats_collector = Collector::new(rx, collector_tx); - stats_collector.collect().await; + stats_collector.collect(addresses).await; }); - let mut pool = - ConnectionPool::from_config(client_server_map.clone(), Reporter::new(tx.clone())).await; - + // Connect to all servers and validate their versions. let server_info = match pool.validate().await { Ok(info) => info, Err(err) => { diff --git a/src/pool.rs b/src/pool.rs index b8a0ecf1f..b53484206 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -38,6 +38,7 @@ impl ConnectionPool { let mut shards = Vec::new(); let mut addresses = Vec::new(); let mut banlist = Vec::new(); + let mut address_id = 0; let mut shard_ids = config .shards .clone() @@ -63,6 +64,7 @@ impl ConnectionPool { }; let address = Address { + id: address_id, host: server.0.clone(), port: server.1.to_string(), role: role, @@ -70,6 +72,8 @@ impl ConnectionPool { shard: shard_idx.parse::().unwrap(), }; + address_id += 1; + if role == Role::Replica { replica_number += 1; } @@ -121,9 +125,13 @@ impl ConnectionPool { pub async fn validate(&mut self) -> Result { let mut server_infos = Vec::new(); + let stats = self.stats.clone(); for shard in 0..self.shards() { for _ in 0..self.servers(shard) { - let connection = match self.get(shard, None).await { + // To keep stats consistent. + let fake_process_id = 0; + + let connection = match self.get(shard, None, fake_process_id).await { Ok(conn) => conn, Err(err) => { error!("Shard {} down or misconfigured: {:?}", shard, err); @@ -137,6 +145,8 @@ impl ConnectionPool { let server_info = server.server_info(); + stats.client_disconnecting(fake_process_id, address.id); + if server_infos.len() > 0 { // Compare against the last server checked. if server_info != server_infos[server_infos.len() - 1] { @@ -165,6 +175,7 @@ impl ConnectionPool { &mut self, shard: usize, role: Option, + process_id: i32, ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { let now = Instant::now(); let addresses = &self.addresses[shard]; @@ -200,6 +211,8 @@ impl ConnectionPool { let index = self.round_robin % addresses.len(); let address = &addresses[index]; + self.stats.client_waiting(process_id, address.id); + // Make sure you're getting a primary or a replica // as per request. If no specific role is requested, the first // available will be chosen. @@ -219,6 +232,9 @@ impl ConnectionPool { Err(err) => { error!("Banning replica {}, error: {:?}", index, err); self.ban(address, shard); + self.stats.client_disconnecting(process_id, address.id); + self.stats + .checkout_time(now.elapsed().as_micros(), address.id); continue; } }; @@ -227,7 +243,7 @@ impl ConnectionPool { let server = &mut *conn; let healthcheck_timeout = get_config().general.healthcheck_timeout; - self.stats.server_tested(server.process_id()); + self.stats.server_tested(server.process_id(), address.id); match tokio::time::timeout( tokio::time::Duration::from_millis(healthcheck_timeout), @@ -238,8 +254,9 @@ impl ConnectionPool { // Check if health check succeeded Ok(res) => match res { Ok(_) => { - self.stats.checkout_time(now.elapsed().as_micros()); - self.stats.server_idle(conn.process_id()); + self.stats + .checkout_time(now.elapsed().as_micros(), address.id); + self.stats.server_idle(conn.process_id(), address.id); return Ok((conn, address.clone())); } Err(_) => { @@ -248,6 +265,9 @@ impl ConnectionPool { server.mark_bad(); self.ban(address, shard); + self.stats.client_disconnecting(process_id, address.id); + self.stats + .checkout_time(now.elapsed().as_micros(), address.id); continue; } }, @@ -258,6 +278,9 @@ impl ConnectionPool { server.mark_bad(); self.ban(address, shard); + self.stats.client_disconnecting(process_id, address.id); + self.stats + .checkout_time(now.elapsed().as_micros(), address.id); continue; } } @@ -395,13 +418,14 @@ impl ManageConnection for ServerPool { async fn connect(&self) -> Result { info!( "Creating a new connection to {:?} using user {:?}", - self.address, self.user.name + self.address.name(), + self.user.name ); // Put a temporary process_id into the stats // for server login. let process_id = rand::random::(); - self.stats.server_login(process_id); + self.stats.server_login(process_id, self.address.id); match Server::startup( &self.address, @@ -414,12 +438,12 @@ impl ManageConnection for ServerPool { { Ok(conn) => { // Remove the temporary process_id from the stats. - self.stats.server_disconnecting(process_id); + self.stats.server_disconnecting(process_id, self.address.id); Ok(conn) } Err(err) => { // Remove the temporary process_id from the stats. - self.stats.server_disconnecting(process_id); + self.stats.server_disconnecting(process_id, self.address.id); Err(err) } } diff --git a/src/query_router.rs b/src/query_router.rs index 737c4d13f..daa917142 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -12,7 +12,7 @@ use sqlparser::parser::Parser; const CUSTOM_SQL_REGEXES: [&str; 5] = [ r"(?i)^ *SET SHARDING KEY TO '?([0-9]+)'? *;? *$", - r"(?i)^ *SET SHARD TO '?([0-9]+)'? *;? *$", + r"(?i)^ *SET SHARD TO '?([0-9]+|ANY)'? *;? *$", r"(?i)^ *SHOW SHARD *;? *$", r"(?i)^ *SET SERVER ROLE TO '(PRIMARY|REPLICA|ANY|AUTO|DEFAULT)' *;? *$", r"(?i)^ *SHOW SERVER ROLE *;? *$", @@ -192,7 +192,10 @@ impl QueryRouter { } Command::SetShard => { - self.active_shard = Some(value.parse::().unwrap()); + self.active_shard = match value.to_ascii_uppercase().as_ref() { + "ANY" => Some(rand::random::() % self.shards), + _ => Some(value.parse::().unwrap()), + }; } Command::SetServerRole => { diff --git a/src/server.rs b/src/server.rs index 934fc9534..4ef9422ad 100644 --- a/src/server.rs +++ b/src/server.rs @@ -268,7 +268,7 @@ impl Server { /// Send messages to the server from the client. pub async fn send(&mut self, messages: BytesMut) -> Result<(), Error> { - self.stats.data_sent(messages.len()); + self.stats.data_sent(messages.len(), self.address.id); match write_all_half(&mut self.write, messages).await { Ok(_) => Ok(()), @@ -374,7 +374,7 @@ impl Server { let bytes = self.buffer.clone(); // Keep track of how much data we got from the server for stats. - self.stats.data_received(bytes.len()); + self.stats.data_received(bytes.len(), self.address.id); // Clear the buffer for next query. self.buffer.clear(); @@ -470,7 +470,8 @@ impl Drop for Server { /// the socket is in non-blocking mode, so it may not be ready /// for a write. fn drop(&mut self) { - self.stats.server_disconnecting(self.process_id()); + self.stats + .server_disconnecting(self.process_id(), self.address.id); let mut bytes = BytesMut::with_capacity(4); bytes.put_u8(b'X'); diff --git a/src/stats.rs b/src/stats.rs index 99f709dbd..7d4b025d4 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,15 +1,13 @@ -use log::{debug, info}; +use log::info; use once_cell::sync::Lazy; use parking_lot::Mutex; -use statsd::Client; use tokio::sync::mpsc::{Receiver, Sender}; use std::collections::HashMap; -use crate::config::get_config; - // Stats used in SHOW STATS -static LATEST_STATS: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); +static LATEST_STATS: Lazy>>> = + Lazy::new(|| Mutex::new(HashMap::new())); static STAT_PERIOD: u64 = 15000; //15 seconds #[derive(Debug, Clone, Copy)] @@ -28,7 +26,8 @@ enum EventName { ServerTested, ServerLogin, ServerDisconnecting, - FlushStatsToStatsD, + UpdateStats, + UpdateAverages, } #[derive(Debug)] @@ -36,6 +35,7 @@ pub struct Event { name: EventName, value: i64, process_id: Option, + address_id: usize, } #[derive(Clone, Debug)] @@ -48,141 +48,155 @@ impl Reporter { Reporter { tx: tx } } - pub fn query(&self) { + pub fn query(&self, address_id: usize) { let event = Event { name: EventName::Query, value: 1, process_id: None, + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn transaction(&self) { + pub fn transaction(&self, address_id: usize) { let event = Event { name: EventName::Transaction, value: 1, process_id: None, + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn data_sent(&self, amount: usize) { + pub fn data_sent(&self, amount: usize, address_id: usize) { let event = Event { name: EventName::DataSent, value: amount as i64, process_id: None, + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn data_received(&self, amount: usize) { + pub fn data_received(&self, amount: usize, address_id: usize) { let event = Event { name: EventName::DataReceived, value: amount as i64, process_id: None, + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn checkout_time(&self, ms: u128) { + pub fn checkout_time(&self, ms: u128, address_id: usize) { let event = Event { name: EventName::CheckoutTime, value: ms as i64, process_id: None, + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn client_waiting(&self, process_id: i32) { + pub fn client_waiting(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ClientWaiting, value: 1, process_id: Some(process_id), + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn client_active(&self, process_id: i32) { + pub fn client_active(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ClientActive, value: 1, process_id: Some(process_id), + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn client_idle(&self, process_id: i32) { + pub fn client_idle(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ClientIdle, value: 1, process_id: Some(process_id), + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn client_disconnecting(&self, process_id: i32) { + pub fn client_disconnecting(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ClientDisconnecting, value: 1, process_id: Some(process_id), + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn server_active(&self, process_id: i32) { + pub fn server_active(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerActive, value: 1, process_id: Some(process_id), + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn server_idle(&self, process_id: i32) { + pub fn server_idle(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerIdle, value: 1, process_id: Some(process_id), + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn server_login(&self, process_id: i32) { + pub fn server_login(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerLogin, value: 1, process_id: Some(process_id), + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn server_tested(&self, process_id: i32) { + pub fn server_tested(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerTested, value: 1, process_id: Some(process_id), + address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn server_disconnecting(&self, process_id: i32) { + pub fn server_disconnecting(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerDisconnecting, value: 1, process_id: Some(process_id), + address_id: address_id, }; let _ = self.tx.try_send(event); @@ -192,22 +206,17 @@ impl Reporter { pub struct Collector { rx: Receiver, tx: Sender, - client: Client, } impl Collector { pub fn new(rx: Receiver, tx: Sender) -> Collector { - Collector { - rx, - tx, - client: Client::new(&get_config().general.statsd_address, "pgcat").unwrap(), - } + Collector { rx, tx } } - pub async fn collect(&mut self) { + pub async fn collect(&mut self, addresses: usize) { info!("Events reporter started"); - let mut stats = HashMap::from([ + let stats_template = HashMap::from([ ("total_query_count", 0), ("total_xact_count", 0), ("total_sent", 0), @@ -232,25 +241,47 @@ impl Collector { ("sv_tested", 0), ]); + let mut stats = HashMap::new(); + // Stats saved after each iteration of the flush event. Used in calculation // of averages in the last flush period. - let mut old_stats: HashMap = HashMap::new(); + let mut old_stats: HashMap> = HashMap::new(); // Track which state the client and server are at any given time. - let mut client_server_states: HashMap = HashMap::new(); + let mut client_server_states: HashMap> = HashMap::new(); // Flush stats to StatsD and calculate averages every 15 seconds. + let tx = self.tx.clone(); + tokio::task::spawn(async move { + let mut interval = + tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD / 15)); + loop { + interval.tick().await; + for address_id in 0..addresses { + let _ = tx.try_send(Event { + name: EventName::UpdateStats, + value: 0, + process_id: None, + address_id: address_id, + }); + } + } + }); + let tx = self.tx.clone(); tokio::task::spawn(async move { let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD)); loop { interval.tick().await; - let _ = tx.try_send(Event { - name: EventName::FlushStatsToStatsD, - value: 0, - process_id: None, - }); + for address_id in 0..addresses { + let _ = tx.try_send(Event { + name: EventName::UpdateAverages, + value: 0, + process_id: None, + address_id: address_id, + }); + } } }); @@ -264,6 +295,14 @@ impl Collector { } }; + let stats = stats + .entry(stat.address_id) + .or_insert(stats_template.clone()); + let client_server_states = client_server_states + .entry(stat.address_id) + .or_insert(HashMap::new()); + let old_stats = old_stats.entry(stat.address_id).or_insert(HashMap::new()); + // Some are counters, some are gauges... match stat.name { EventName::Query => { @@ -320,9 +359,9 @@ impl Collector { client_server_states.remove(&stat.process_id.unwrap()); } - EventName::FlushStatsToStatsD => { + EventName::UpdateStats => { // Calculate connection states - for (_, state) in &client_server_states { + for (_, state) in client_server_states.iter() { match state { EventName::ClientActive => { let counter = stats.entry("cl_active").or_insert(0); @@ -334,11 +373,6 @@ impl Collector { *counter += 1; } - EventName::ClientIdle => { - let counter = stats.entry("cl_idle").or_insert(0); - *counter += 1; - } - EventName::ServerIdle => { let counter = stats.entry("sv_idle").or_insert(0); *counter += 1; @@ -359,39 +393,20 @@ impl Collector { *counter += 1; } + EventName::ClientIdle => { + let counter = stats.entry("cl_idle").or_insert(0); + *counter += 1; + } + _ => unreachable!(), }; } - // Calculate averages - for stat in &[ - "avg_query_count", - "avgxact_count", - "avg_sent", - "avg_received", - "avg_wait_time", - ] { - let total_name = stat.replace("avg_", "total_"); - let old_value = old_stats.entry(total_name.clone()).or_insert(0); - let new_value = stats.get(total_name.as_str()).unwrap_or(&0).to_owned(); - let avg = (new_value - *old_value) / (STAT_PERIOD as i64 / 1_000); // Avg / second - - stats.insert(stat, avg); - *old_value = new_value; - } - - debug!("{:?}", stats); - // Update latest stats used in SHOW STATS let mut guard = LATEST_STATS.lock(); - for (key, value) in &stats { - guard.insert(key.to_string(), value.clone()); - } - - let mut pipeline = self.client.pipeline(); - for (key, value) in stats.iter() { - pipeline.gauge(key, *value as f64); + let entry = guard.entry(stat.address_id).or_insert(HashMap::new()); + entry.insert(key.to_string(), value.clone()); } // These are re-calculated every iteration of the loop, so we don't want to add values @@ -409,14 +424,31 @@ impl Collector { ] { stats.insert(stat, 0); } + } - pipeline.send(&self.client); + EventName::UpdateAverages => { + // Calculate averages + for stat in &[ + "avg_query_count", + "avgxact_count", + "avg_sent", + "avg_received", + "avg_wait_time", + ] { + let total_name = stat.replace("avg_", "total_"); + let old_value = old_stats.entry(total_name.clone()).or_insert(0); + let new_value = stats.get(total_name.as_str()).unwrap_or(&0).to_owned(); + let avg = (new_value - *old_value) / (STAT_PERIOD as i64 / 1_000); // Avg / second + + stats.insert(stat, avg); + *old_value = new_value; + } } }; } } } -pub fn get_stats() -> HashMap { +pub fn get_stats() -> HashMap> { LATEST_STATS.lock().clone() } diff --git a/tests/pgbench/simple.sql b/tests/pgbench/simple.sql new file mode 100644 index 000000000..0a283bae4 --- /dev/null +++ b/tests/pgbench/simple.sql @@ -0,0 +1,28 @@ + +-- \setrandom aid 1 :naccounts +\set aid random(1, 100000) +-- \setrandom bid 1 :nbranches +\set bid random(1, 100000) +-- \setrandom tid 1 :ntellers +\set tid random(1, 100000) +-- \setrandom delta -5000 5000 +\set delta random(-5000,5000) + +\set shard random(0, 2) + +SET SHARD TO :shard; + +BEGIN; + +UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid; + +SELECT abalance FROM pgbench_accounts WHERE aid = :aid; + +UPDATE pgbench_tellers SET tbalance = tbalance + :delta WHERE tid = :tid; + +UPDATE pgbench_branches SET bbalance = bbalance + :delta WHERE bid = :bid; + +INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :aid, :delta, CURRENT_TIMESTAMP); + +END; + From c54c9ad18abc5b72e3c487ab0997bec88145f6ca Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 7 Mar 2022 23:05:40 -0800 Subject: [PATCH 125/551] docs and remove Option (#58) * docs and remove Option * lint --- src/client.rs | 10 ++--- src/pool.rs | 8 ++-- src/server.rs | 6 ++- src/stats.rs | 119 ++++++++++++++++++++++++++++++++++++++------------ 4 files changed, 104 insertions(+), 39 deletions(-) diff --git a/src/client.rs b/src/client.rs index 0fc9c4749..dafc97d2e 100644 --- a/src/client.rs +++ b/src/client.rs @@ -407,12 +407,12 @@ impl Client { } // Report query executed statistics. - self.stats.query(address.id); + self.stats.query(self.process_id, address.id); // The transaction is over, we can release the connection back to the pool. if !server.in_transaction() { // Report transaction executed statistics. - self.stats.transaction(address.id); + self.stats.transaction(self.process_id, address.id); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -493,12 +493,12 @@ impl Client { } // Report query executed statistics. - self.stats.query(address.id); + self.stats.query(self.process_id, address.id); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if !server.in_transaction() { - self.stats.transaction(address.id); + self.stats.transaction(self.process_id, address.id); if self.transaction_mode { self.stats.server_idle(server.process_id(), address.id); @@ -532,7 +532,7 @@ impl Client { // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if !server.in_transaction() { - self.stats.transaction(address.id); + self.stats.transaction(self.process_id, address.id); if self.transaction_mode { self.stats.server_idle(server.process_id(), address.id); diff --git a/src/pool.rs b/src/pool.rs index b53484206..046a6811a 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -234,7 +234,7 @@ impl ConnectionPool { self.ban(address, shard); self.stats.client_disconnecting(process_id, address.id); self.stats - .checkout_time(now.elapsed().as_micros(), address.id); + .checkout_time(now.elapsed().as_micros(), process_id, address.id); continue; } }; @@ -255,7 +255,7 @@ impl ConnectionPool { Ok(res) => match res { Ok(_) => { self.stats - .checkout_time(now.elapsed().as_micros(), address.id); + .checkout_time(now.elapsed().as_micros(), process_id, address.id); self.stats.server_idle(conn.process_id(), address.id); return Ok((conn, address.clone())); } @@ -267,7 +267,7 @@ impl ConnectionPool { self.ban(address, shard); self.stats.client_disconnecting(process_id, address.id); self.stats - .checkout_time(now.elapsed().as_micros(), address.id); + .checkout_time(now.elapsed().as_micros(), process_id, address.id); continue; } }, @@ -280,7 +280,7 @@ impl ConnectionPool { self.ban(address, shard); self.stats.client_disconnecting(process_id, address.id); self.stats - .checkout_time(now.elapsed().as_micros(), address.id); + .checkout_time(now.elapsed().as_micros(), process_id, address.id); continue; } } diff --git a/src/server.rs b/src/server.rs index 4ef9422ad..aaab47c0c 100644 --- a/src/server.rs +++ b/src/server.rs @@ -268,7 +268,8 @@ impl Server { /// Send messages to the server from the client. pub async fn send(&mut self, messages: BytesMut) -> Result<(), Error> { - self.stats.data_sent(messages.len(), self.address.id); + self.stats + .data_sent(messages.len(), self.process_id, self.address.id); match write_all_half(&mut self.write, messages).await { Ok(_) => Ok(()), @@ -374,7 +375,8 @@ impl Server { let bytes = self.buffer.clone(); // Keep track of how much data we got from the server for stats. - self.stats.data_received(bytes.len(), self.address.id); + self.stats + .data_received(bytes.len(), self.process_id, self.address.id); // Clear the buffer for next query. self.buffer.clear(); diff --git a/src/stats.rs b/src/stats.rs index 7d4b025d4..1b7ccda21 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,15 +1,20 @@ +/// Statistics and reporting. use log::info; use once_cell::sync::Lazy; use parking_lot::Mutex; -use tokio::sync::mpsc::{Receiver, Sender}; - use std::collections::HashMap; +use tokio::sync::mpsc::{Receiver, Sender}; -// Stats used in SHOW STATS +// Latest stats updated every second; used in SHOW STATS and other admin commands. static LATEST_STATS: Lazy>>> = Lazy::new(|| Mutex::new(HashMap::new())); -static STAT_PERIOD: u64 = 15000; //15 seconds +// Statistics period used for average calculations. +// 15 seconds. +static STAT_PERIOD: u64 = 15000; + +/// The names for the events reported +/// to the statistics collector. #[derive(Debug, Clone, Copy)] enum EventName { CheckoutTime, @@ -30,172 +35,218 @@ enum EventName { UpdateAverages, } +/// Event data sent to the collector +/// from clients and servers. #[derive(Debug)] pub struct Event { + /// The name of the event being reported. name: EventName, + + /// The value being reported. Meaning differs based on event name. value: i64, - process_id: Option, + + /// The client or server connection reporting the event. + process_id: i32, + + /// The server the client is connected to. address_id: usize, } +/// The statistics reporter. An instance is given +/// to each possible source of statistics, +/// e.g. clients, servers, connection pool. #[derive(Clone, Debug)] pub struct Reporter { tx: Sender, } impl Reporter { + /// Create a new Reporter instance. pub fn new(tx: Sender) -> Reporter { Reporter { tx: tx } } - pub fn query(&self, address_id: usize) { + /// Report a query executed by a client against + /// a server identified by the `address_id`. + pub fn query(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::Query, value: 1, - process_id: None, + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn transaction(&self, address_id: usize) { + /// Report a transaction executed by a client against + /// a server identified by the `address_id`. + pub fn transaction(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::Transaction, value: 1, - process_id: None, + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn data_sent(&self, amount: usize, address_id: usize) { + /// Report data sent to a server identified by `address_id`. + /// The `amount` is measured in bytes. + pub fn data_sent(&self, amount: usize, process_id: i32, address_id: usize) { let event = Event { name: EventName::DataSent, value: amount as i64, - process_id: None, + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn data_received(&self, amount: usize, address_id: usize) { + /// Report data received from a server identified by `address_id`. + /// The `amount` is measured in bytes. + pub fn data_received(&self, amount: usize, process_id: i32, address_id: usize) { let event = Event { name: EventName::DataReceived, value: amount as i64, - process_id: None, + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } - pub fn checkout_time(&self, ms: u128, address_id: usize) { + /// Time spent waiting to get a healthy connection from the pool + /// for a server identified by `address_id`. + /// Measured in milliseconds. + pub fn checkout_time(&self, ms: u128, process_id: i32, address_id: usize) { let event = Event { name: EventName::CheckoutTime, value: ms as i64, - process_id: None, + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } + /// Reports a client identified by `process_id` waiting for a connection + /// to a server identified by `address_id`. pub fn client_waiting(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ClientWaiting, value: 1, - process_id: Some(process_id), + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } + /// Reports a client identified by `process_id` is done waiting for a connection + /// to a server identified by `address_id` and is about to query the server. pub fn client_active(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ClientActive, value: 1, - process_id: Some(process_id), + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } + /// Reports a client identified by `process_id` is done querying the server + /// identified by `address_id` and is no longer active. pub fn client_idle(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ClientIdle, value: 1, - process_id: Some(process_id), + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } + /// Reports a client identified by `process_id` is disconecting from the pooler. + /// The last server it was connected to is identified by `address_id`. pub fn client_disconnecting(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ClientDisconnecting, value: 1, - process_id: Some(process_id), + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } + /// Reports a server connection identified by `process_id` for + /// a configured server identified by `address_id` is actively used + /// by a client. pub fn server_active(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerActive, value: 1, - process_id: Some(process_id), + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } + /// Reports a server connection identified by `process_id` for + /// a configured server identified by `address_id` is no longer + /// actively used by a client and is now idle. pub fn server_idle(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerIdle, value: 1, - process_id: Some(process_id), + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } + /// Reports a server connection identified by `process_id` for + /// a configured server identified by `address_id` is attempting + /// to login. pub fn server_login(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerLogin, value: 1, - process_id: Some(process_id), + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } + /// Reports a server connection identified by `process_id` for + /// a configured server identified by `address_id` is being + /// tested before being given to a client. pub fn server_tested(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerTested, value: 1, - process_id: Some(process_id), + process_id: process_id, address_id: address_id, }; let _ = self.tx.try_send(event); } + /// Reports a server connection identified by `process_id` is disconecting from the pooler. + /// The configured server it was connected to is identified by `address_id`. pub fn server_disconnecting(&self, process_id: i32, address_id: usize) { let event = Event { name: EventName::ServerDisconnecting, value: 1, - process_id: Some(process_id), + process_id: process_id, address_id: address_id, }; @@ -203,16 +254,26 @@ impl Reporter { } } +/// The statistics collector which is receiving statistics +/// from clients, servers, and the connection pool. There is +/// only one collector (kind of like a singleton). +/// The collector can trigger events on its own, e.g. +/// it updates aggregates every second and averages every +/// 15 seconds. pub struct Collector { rx: Receiver, tx: Sender, } impl Collector { + /// Create a new collector instance. There should only be one instance + /// at a time. This is ensured by mpsc which allows only one receiver. pub fn new(rx: Receiver, tx: Sender) -> Collector { Collector { rx, tx } } + /// The statistics collection handler. It will collect statistics + /// for `address_id`s starting at 0 up to `addresses`. pub async fn collect(&mut self, addresses: usize) { info!("Events reporter started"); @@ -261,7 +322,7 @@ impl Collector { let _ = tx.try_send(Event { name: EventName::UpdateStats, value: 0, - process_id: None, + process_id: -1, address_id: address_id, }); } @@ -278,7 +339,7 @@ impl Collector { let _ = tx.try_send(Event { name: EventName::UpdateAverages, value: 0, - process_id: None, + process_id: -1, address_id: address_id, }); } @@ -352,11 +413,11 @@ impl Collector { | EventName::ServerIdle | EventName::ServerTested | EventName::ServerLogin => { - client_server_states.insert(stat.process_id.unwrap(), stat.name); + client_server_states.insert(stat.process_id, stat.name); } EventName::ClientDisconnecting | EventName::ServerDisconnecting => { - client_server_states.remove(&stat.process_id.unwrap()); + client_server_states.remove(&stat.process_id); } EventName::UpdateStats => { @@ -449,6 +510,8 @@ impl Collector { } } +/// Get a snapshot of statistics. Updated once a second +/// by the `Collector`. pub fn get_stats() -> HashMap> { LATEST_STATS.lock().clone() } From f5ca564e2aa8a4b277e83af99d0ea541d58c118f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Mar 2022 17:18:48 -0800 Subject: [PATCH 126/551] Handle SIGTERM. Add docker-compose.yml (#59) * docker-compsoe * remove statsd config * readme --- .circleci/pgcat.toml | 3 -- Dockerfile | 2 +- README.md | 14 +++-- docker-compose.yml | 16 ++++++ examples/docker/pgcat.toml | 105 +++++++++++++++++++++++++++++++++++++ pgcat.toml | 3 -- src/config.rs | 6 --- src/main.rs | 14 +++-- 8 files changed, 139 insertions(+), 24 deletions(-) create mode 100644 docker-compose.yml create mode 100644 examples/docker/pgcat.toml diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 370b06b18..d1dd19db5 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -29,9 +29,6 @@ healthcheck_timeout = 100 # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # Seconds -# Stats will be sent here -statsd_address = "127.0.0.1:8125" - # # User to use for authentication against the server. [user] diff --git a/Dockerfile b/Dockerfile index cdd0e557c..536647478 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,4 +8,4 @@ COPY --from=builder /app/target/release/pgcat /usr/bin/pgcat COPY --from=builder /app/pgcat.toml /etc/pgcat/pgcat.toml WORKDIR /etc/pgcat ENV RUST_LOG=info -ENTRYPOINT ["/usr/bin/pgcat"] +CMD ["pgcat"] diff --git a/README.md b/README.md index ab9406167..9524f5ad6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover support. -**Alpha**: looking for alpha testers, see [#35](https://github.com/levkk/pgcat/issues/35). +**Beta**: looking for beta testers, see [#35](https://github.com/levkk/pgcat/issues/35). ## Features | **Feature** | **Status** | **Comments** | @@ -24,8 +24,16 @@ Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover su ## Deployment -See `Dockerfile` for example deployment using Docker. The pooler is configured to spawn 4 workers so 4 CPUs are recommended for optimal performance. -That setting can be adjusted to spawn as many (or as little) workers as needed. +See `Dockerfile` for example deployment using Docker. The pooler is configured to spawn 4 workers so 4 CPUs are recommended for optimal performance. That setting can be adjusted to spawn as many (or as little) workers as needed. + +For quick local example, use the Docker Compose environment provided: + +```bash +docker-compose up + +# In a new terminal: +psql -h 127.0.0.1 -p 6432 -c 'SELECT 1' +``` ### Config diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..510546e61 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3" +services: + postgres: + image: postgres:13 + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: md5 + pgcat: + build: . + command: + - "pgcat" + - "/etc/pgcat/pgcat.toml" + volumes: + - "${PWD}/examples/docker/pgcat.toml:/etc/pgcat/pgcat.toml" + ports: + - "6432:6432" diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml new file mode 100644 index 000000000..92c728a79 --- /dev/null +++ b/examples/docker/pgcat.toml @@ -0,0 +1,105 @@ +# +# PgCat config example. +# + +# +# General pooler settings +[general] + +# What IP to run on, 0.0.0.0 means accessible from everywhere. +host = "0.0.0.0" + +# Port to run on, same as PgBouncer used in this example. +port = 6432 + +# How many connections to allocate per server. +pool_size = 15 + +# Pool mode (see PgBouncer docs for more). +# session: one server connection per connected client +# transaction: one server connection per client transaction +pool_mode = "transaction" + +# How long to wait before aborting a server connection (ms). +connect_timeout = 5000 + +# How much time to give `SELECT 1` health check query to return with a result (ms). +healthcheck_timeout = 1000 + +# For how long to ban a server if it fails a health check (seconds). +ban_time = 60 # Seconds + +# +# User to use for authentication against the server. +[user] +name = "postgres" +password = "postgres" + + +# +# Shards in the cluster +[shards] + +# Shard 0 +[shards.0] + +# [ host, port, role ] +servers = [ + [ "postgres", 5432, "primary" ], + [ "postgres", 5432, "replica" ], + # [ "127.0.1.1", 5432, "replica" ], +] +# Database name (e.g. "postgres") +database = "postgres" + +[shards.1] +# [ host, port, role ] +servers = [ + [ "postgres", 5432, "primary" ], + [ "postgres", 5432, "replica" ], + # [ "127.0.1.1", 5432, "replica" ], +] +database = "postgres" + +[shards.2] +# [ host, port, role ] +servers = [ + [ "postgres", 5432, "primary" ], + [ "postgres", 5432, "replica" ], + # [ "127.0.1.1", 5432, "replica" ], +] +database = "postgres" + + +# Settings for our query routing layer. +[query_router] + +# If the client doesn't specify, route traffic to +# this role by default. +# +# any: round-robin between primary and replicas, +# replica: round-robin between replicas only without touching the primary, +# primary: all queries go to the primary unless otherwise specified. +default_role = "any" + + +# Query parser. If enabled, we'll attempt to parse +# every incoming query to determine if it's a read or a write. +# If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, +# we'll direct it to the primary. +query_parser_enabled = false + +# If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for +# load balancing of read queries. Otherwise, the primary will only be used for write +# queries. The primary can always be explicitely selected with our custom protocol. +primary_reads_enabled = true + +# So what if you wanted to implement a different hashing function, +# or you've already built one and you want this pooler to use it? +# +# Current options: +# +# pg_bigint_hash: PARTITION BY HASH (Postgres hashing function) +# sha1: A hashing function based on SHA1 +# +sharding_function = "pg_bigint_hash" diff --git a/pgcat.toml b/pgcat.toml index 213180237..5311bd74d 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -29,9 +29,6 @@ healthcheck_timeout = 1000 # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # Seconds -# Stats will be sent here -statsd_address = "127.0.0.1:8125" - # # User to use for authentication against the server. [user] diff --git a/src/config.rs b/src/config.rs index cd8b2eb61..87c40a5cc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -103,7 +103,6 @@ pub struct General { pub connect_timeout: u64, pub healthcheck_timeout: u64, pub ban_time: i64, - pub statsd_address: String, } impl Default for General { @@ -116,7 +115,6 @@ impl Default for General { connect_timeout: 5000, healthcheck_timeout: 1000, ban_time: 60, - statsd_address: String::from("127.0.0.1:8125"), } } } @@ -198,10 +196,6 @@ impl From<&Config> for std::collections::HashMap { config.general.healthcheck_timeout.to_string(), ), ("ban_time".to_string(), config.general.ban_time.to_string()), - ( - "statsd_address".to_string(), - config.general.statsd_address.to_string(), - ), ( "default_role".to_string(), config.query_router.default_role.to_string(), diff --git a/src/main.rs b/src/main.rs index ada4551f1..97b198880 100644 --- a/src/main.rs +++ b/src/main.rs @@ -205,16 +205,14 @@ async fn main() { } }); - // Setup shut down sequence - match signal::ctrl_c().await { - Ok(()) => { - info!("Shutting down..."); - } + let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); - Err(err) => { - error!("Unable to listen for shutdown signal: {}", err); - } + tokio::select! { + _ = signal::ctrl_c() => (), + _ = term_signal.recv() => (), }; + + info!("Shutting down..."); } /// Format chrono::Duration to be more human-friendly. From bda96ce9fa91fa72e46fd60d9d094d0e875e20fb Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Mar 2022 17:38:51 -0800 Subject: [PATCH 127/551] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9524f5ad6..edb9885e7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ![PgCat](./pgcat3.png) -Meow. PgBouncer rewritten in Rust, with sharding, load balancing and failover support. +PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover support. **Beta**: looking for beta testers, see [#35](https://github.com/levkk/pgcat/issues/35). From 9a323f138d301728bf087e494c2417aae255914d Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Mar 2022 17:45:54 -0800 Subject: [PATCH 128/551] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index edb9885e7..9a6e0791a 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover su | Load balancing of read queries | :white_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | | Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | | Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | -| Statistics reporting | :white_check_mark: | Statistics similar to PgBouncers are reported via StatsD. | -| Live configuration reloading | :construction_worker: | Reload config with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)`. Not all settings can be reloaded without a restart. | +| Statistics reporting | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. | +| Live configuration reloading | :construction_worker: | Reload supported settings with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)` or `RELOAD` query issued to the admin database. | | Client authentication | :x: :wrench: | On the roadmap; currently all clients are allowed to connect and one user is used to connect to Postgres. | ## Deployment From a9969637782da9644d1be50c15bbdbc0373b26a8 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Mar 2022 17:46:46 -0800 Subject: [PATCH 129/551] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9a6e0791a..7a88c1c53 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover su | Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | | Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | | Statistics reporting | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. | -| Live configuration reloading | :construction_worker: | Reload supported settings with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)` or `RELOAD` query issued to the admin database. | +| Live configuration reloading | :white_check_mark: | Reload supported settings with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)` or `RELOAD` query issued to the admin database. | | Client authentication | :x: :wrench: | On the roadmap; currently all clients are allowed to connect and one user is used to connect to Postgres. | ## Deployment From fca5a4ff9b3f984cd9ec664988c216188ef8c7cc Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Mar 2022 17:48:26 -0800 Subject: [PATCH 130/551] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7a88c1c53..2a3ae9ecb 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,6 @@ psql -h 127.0.0.1 -p 6432 -c 'SELECT 1' | `connect_timeout` | Maximum time to establish a connection to a server (milliseconds). If reached, the server is banned and the next target is attempted. | `5000` | | `healthcheck_timeout` | Maximum time to pass a health check (`SELECT 1`, milliseconds). If reached, the server is banned and the next target is attempted. | `1000` | | `ban_time` | Ban time for a server (seconds). It won't be allowed to serve transactions until the ban expires; failover targets will be used instead. | `60` | -| `statsd_address` | StatsD host and port. Statistics will be sent there every 15 seconds. | `127.0.0.1:8125` | | | | | | **`user`** | | | | `name` | The user name. | `sharding_user` | From 58360e87cc160b43191b08b67b3a04ab8a281768 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Mar 2022 01:33:29 -0800 Subject: [PATCH 131/551] Update README. Comments. Version bump. (#60) * update readme * comments * just a version bump --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 16 ++++--- src/admin.rs | 26 ++++++----- src/client.rs | 111 +++++++++++++++++++++----------------------- src/config.rs | 22 +++++++-- src/constants.rs | 3 +- src/errors.rs | 6 +-- src/main.rs | 20 ++++---- src/messages.rs | 4 +- src/pool.rs | 36 ++++++++------ src/query_router.rs | 39 ++++++++++------ src/server.rs | 45 +++++++++--------- src/sharding.rs | 13 +++++- src/stats.rs | 6 +-- 15 files changed, 196 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ee21380a..fe3c2999b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -352,7 +352,7 @@ dependencies = [ [[package]] name = "pgcat" -version = "0.1.0" +version = "0.1.0-beta2" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 74d4a76a9..d41657ad1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "0.1.0" +version = "0.1.0-beta2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README.md b/README.md index 2a3ae9ecb..12b84c747 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,10 @@ PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover su | Load balancing of read queries | :white_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | | Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | | Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | -| Statistics reporting | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. | -| Live configuration reloading | :white_check_mark: | Reload supported settings with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)` or `RELOAD` query issued to the admin database. | +| Statistics | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. | +| Live configuration reloading | :white_check_mark: | Reload supported settings with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)` or `RELOAD` query issued to the admin database. | | Client authentication | :x: :wrench: | On the roadmap; currently all clients are allowed to connect and one user is used to connect to Postgres. | +| Admin database | :white_check_mark: | The admin database, similar to PgBouncer's, allows to query for statistics and reload the configuration. | ## Deployment @@ -89,7 +90,7 @@ See [sharding README](./tests/sharding/README.md) for sharding logic testing. | Load balancing | :white_check_mark: | :white_check_mark: | We could test this by emitting statistics for each replica and compare them. | | Failover | :white_check_mark: | :white_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing is using Toxiproxy. | | Sharding | :white_check_mark: | :white_check_mark: | See `tests/sharding` and `tests/ruby` for an Rails/ActiveRecord example. | -| Statistics reporting | :x: | :white_check_mark: | Run `nc -l -u 8125` and watch the stats come in every 15 seconds. | +| Statistics | :white_check_mark: | :white_check_mark: | Query the admin database with `psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS'`. | | Live config reloading | :white_check_mark: | :white_check_mark: | Run `kill -s SIGHUP $(pgrep pgcat)` and watch the config reload. | ## Usage @@ -232,11 +233,15 @@ SELECT * FROM users WHERE email = 'test@example.com'; -- shard setting lasts unt ### Statistics reporting -Stats are reported using StatsD every 15 seconds. The address is configurable with `statsd_address`, the default is `127.0.0.1:8125`. The stats are very similar to what Pgbouncer reports and the names are kept to be comparable. +The stats are very similar to what Pgbouncer reports and the names are kept to be comparable. They are accessible by querying the admin database `pgcat`, and `pgbouncer` for compatibility. + +``` +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' +``` ### Live configuration reloading -The config can be reloaded by sending a `kill -s SIGHUP` to the process. Not all settings are currently supported by live reload: +The config can be reloaded by sending a `kill -s SIGHUP` to the process or by querying `RELOAD` to the admin database. Not all settings are currently supported by live reload: | **Config** | **Requires restart** | |-------------------------|----------------------| @@ -246,7 +251,6 @@ The config can be reloaded by sending a `kill -s SIGHUP` to the process. Not all | `connect_timeout` | yes | | `healthcheck_timeout` | no | | `ban_time` | no | -| `statsd_address` | yes | | `user` | yes | | `shards` | yes | | `default_role` | no | diff --git a/src/admin.rs b/src/admin.rs index 8f94b5db1..c4675703d 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,8 +1,8 @@ +/// Admin database. use bytes::{Buf, BufMut, BytesMut}; use log::{info, trace}; -use tokio::net::tcp::OwnedWriteHalf; - use std::collections::HashMap; +use tokio::net::tcp::OwnedWriteHalf; use crate::config::{get_config, parse}; use crate::errors::Error; @@ -10,7 +10,7 @@ use crate::messages::*; use crate::pool::ConnectionPool; use crate::stats::get_stats; -/// Handle admin client +/// Handle admin client. pub async fn handle_admin( stream: &mut OwnedWriteHalf, mut query: BytesMut, @@ -58,7 +58,7 @@ pub async fn handle_admin( } } -/// SHOW LISTS +/// Column-oriented statistics. async fn show_lists(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { let stats = get_stats(); @@ -125,7 +125,7 @@ async fn show_lists(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul write_all_half(stream, res).await } -/// SHOW VERSION +/// Show PgCat version. async fn show_version(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let mut res = BytesMut::new(); @@ -140,7 +140,7 @@ async fn show_version(stream: &mut OwnedWriteHalf) -> Result<(), Error> { write_all_half(stream, res).await } -/// SHOW POOLS +/// Show utilization of connection pools for each shard and replicas. async fn show_pools(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { let stats = get_stats(); let config = { @@ -189,6 +189,7 @@ async fn show_pools(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul res.put(command_complete("SHOW")); + // ReadyForQuery res.put_u8(b'Z'); res.put_i32(5); res.put_u8(b'I'); @@ -196,7 +197,7 @@ async fn show_pools(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul write_all_half(stream, res).await } -/// SHOW DATABASES +/// Show shards and replicas. async fn show_databases(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { let guard = get_config(); let config = &*guard.clone(); @@ -221,7 +222,6 @@ async fn show_databases(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> R let mut res = BytesMut::new(); - // RowDescription res.put(row_description(&columns)); for shard in 0..pool.shards() { @@ -265,7 +265,7 @@ async fn ignore_set(stream: &mut OwnedWriteHalf) -> Result<(), Error> { custom_protocol_response_ok(stream, "SET").await } -/// RELOAD +/// Reload the configuration file without restarting the process. async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { info!("Reloading config"); @@ -280,7 +280,6 @@ async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let mut res = BytesMut::new(); - // CommandComplete res.put(command_complete("RELOAD")); // ReadyForQuery @@ -291,13 +290,14 @@ async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { write_all_half(stream, res).await } +/// Shows current configuration. async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { let guard = get_config(); let config = &*guard.clone(); let config: HashMap = config.into(); drop(guard); - // Configs that cannot be changed dynamically. + // Configs that cannot be changed without restarting. let immutables = ["host", "port", "connect_timeout"]; // Columns @@ -327,6 +327,7 @@ async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { res.put(command_complete("SHOW")); + // ReadyForQuery res.put_u8(b'Z'); res.put_i32(5); res.put_u8(b'I'); @@ -334,7 +335,7 @@ async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { write_all_half(stream, res).await } -/// SHOW STATS +/// Show shard and replicas statistics. async fn show_stats(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { let columns = vec![ ("database", DataType::Text), @@ -378,6 +379,7 @@ async fn show_stats(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul res.put(command_complete("SHOW")); + // ReadyForQuery res.put_u8(b'Z'); res.put_i32(5); res.put_u8(b'I'); diff --git a/src/client.rs b/src/client.rs index dafc97d2e..87f621562 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,16 +1,13 @@ -/// Implementation of the PostgreSQL client. -/// We are pretending to the server in this scenario, -/// and this module implements that. +/// Handle clients by pretending to be a PostgreSQL server. use bytes::{Buf, BufMut, BytesMut}; use log::{debug, error, trace}; +use std::collections::HashMap; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, TcpStream, }; -use std::collections::HashMap; - use crate::admin::handle_admin; use crate::config::get_config; use crate::constants::*; @@ -23,53 +20,52 @@ use crate::stats::Reporter; /// The client state. One of these is created per client. pub struct Client { - // The reads are buffered (8K by default). + /// The reads are buffered (8K by default). read: BufReader, - // We buffer the writes ourselves because we know the protocol - // better than a stock buffer. + /// We buffer the writes ourselves because we know the protocol + /// better than a stock buffer. write: OwnedWriteHalf, - // Internal buffer, where we place messages until we have to flush - // them to the backend. + /// Internal buffer, where we place messages until we have to flush + /// them to the backend. buffer: BytesMut, - // The client was started with the sole reason to cancel another running query. + /// The client was started with the sole reason to cancel another running query. cancel_mode: bool, - // In transaction mode, the connection is released after each transaction. - // Session mode has slightly higher throughput per client, but lower capacity. + /// In transaction mode, the connection is released after each transaction. + /// Session mode has slightly higher throughput per client, but lower capacity. transaction_mode: bool, - // For query cancellation, the client is given a random process ID and secret on startup. + /// For query cancellation, the client is given a random process ID and secret on startup. process_id: i32, secret_key: i32, - // Clients are mapped to servers while they use them. This allows a client - // to connect and cancel a query. + /// Clients are mapped to servers while they use them. This allows a client + /// to connect and cancel a query. client_server_map: ClientServerMap, - // Client parameters, e.g. user, client_encoding, etc. + /// Client parameters, e.g. user, client_encoding, etc. #[allow(dead_code)] parameters: HashMap, - // Statistics + /// Statistics stats: Reporter, - // Clients want to talk to admin + /// Clients want to talk to admin database. admin: bool, - // Last address the client talked to + /// Last address the client talked to. last_address_id: Option, - // Last server process id we talked to + /// Last server process id we talked to. last_server_id: Option, } impl Client { - /// Given a TCP socket, trick the client into thinking we are - /// the Postgres server. Perform the authentication and place - /// the client in query-ready mode. + /// Perform client startup sequence. + /// See docs: pub async fn startup( mut stream: TcpStream, client_server_map: ClientServerMap, @@ -82,14 +78,12 @@ impl Client { loop { trace!("Waiting for StartupMessage"); - // Could be StartupMessage or SSLRequest - // which makes this variable length. + // Could be StartupMessage, SSLRequest or CancelRequest. let len = match stream.read_i32().await { Ok(len) => len, Err(_) => return Err(Error::ClientBadStartup), }; - // Read whatever is left. let mut startup = vec![0u8; len as usize - 4]; match stream.read_exact(&mut startup).await { @@ -189,7 +183,7 @@ impl Client { } } - /// Client loop. We handle all messages between the client and the database here. + /// Handle a connected and authenticated client. pub async fn handle(&mut self, mut pool: ConnectionPool) -> Result<(), Error> { // The client wants to cancel a query it has issued previously. if self.cancel_mode { @@ -225,14 +219,14 @@ impl Client { // Our custom protocol loop. // We expect the client to either start a transaction with regular queries - // or issue commands for our sharding and server selection protocols. + // or issue commands for our sharding and server selection protocol. loop { trace!("Client idle, waiting for message"); // Read a complete message from the client, which normally would be // either a `Q` (query) or `P` (prepare, extended protocol). // We can parse it here before grabbing a server from the pool, - // in case the client is sending some control messages, e.g. + // in case the client is sending some custom protocol messages, e.g. // SET SHARDING KEY TO 'bigint'; let mut message = read_message(&mut self.read).await?; @@ -242,43 +236,48 @@ impl Client { return Ok(()); } - // Handle admin database real quick + // Handle admin database queries. if self.admin { trace!("Handling admin command"); handle_admin(&mut self.write, message, pool.clone()).await?; continue; } - // Handle all custom protocol commands here. + // Handle all custom protocol commands, if any. match query_router.try_execute_command(message.clone()) { - // Normal query + // Normal query, not a custom command. None => { + // Attempt to infer which server we want to query, i.e. primary or replica. if query_router.query_parser_enabled() && query_router.role() == None { query_router.infer_role(message.clone()); } } + // SET SHARD TO Some((Command::SetShard, _)) => { - custom_protocol_response_ok(&mut self.write, &format!("SET SHARD")).await?; + custom_protocol_response_ok(&mut self.write, "SET SHARD").await?; continue; } + // SET SHARDING KEY TO Some((Command::SetShardingKey, _)) => { - custom_protocol_response_ok(&mut self.write, &format!("SET SHARDING KEY")) - .await?; + custom_protocol_response_ok(&mut self.write, "SET SHARDING KEY").await?; continue; } + // SET SERVER ROLE TO Some((Command::SetServerRole, _)) => { custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; continue; } + // SHOW SERVER ROLE Some((Command::ShowServerRole, value)) => { show_response(&mut self.write, "server role", &value).await?; continue; } + // SHOW SHARD Some((Command::ShowShard, value)) => { show_response(&mut self.write, "shard", &value).await?; continue; @@ -290,7 +289,7 @@ impl Client { error_response( &mut self.write, &format!( - "shard '{}' is more than configured '{}'", + "shard {} is more than configured {}", query_router.shard(), pool.shards() ), @@ -301,7 +300,7 @@ impl Client { debug!("Waiting for connection from pool"); - // Grab a server from the pool: the client issued a regular query. + // Grab a server from the pool. let connection = match pool .get(query_router.shard(), query_router.role(), self.process_id) .await @@ -322,18 +321,18 @@ impl Client { let address = connection.1; let server = &mut *reference; - // Claim this server as mine for query cancellation. + // Server is assigned to the client in case the client wants to + // cancel a query later. server.claim(self.process_id, self.secret_key); - // "disconnect" from the previous server stats-wise + // Update statistics. if let Some(last_address_id) = self.last_address_id { self.stats .client_disconnecting(self.process_id, last_address_id); } - - // Client active & server active self.stats.client_active(self.process_id, address.id); self.stats.server_active(server.process_id(), address.id); + self.last_address_id = Some(address.id); self.last_server_id = Some(server.process_id()); @@ -346,6 +345,9 @@ impl Client { // Transaction loop. Multiple queries can be issued by the client here. // The connection belongs to the client until the transaction is over, // or until the client disconnects if we are in session mode. + // + // If the client is in session mode, no more custom protocol + // commands will be accepted. loop { let mut message = if message.len() == 0 { trace!("Waiting for message inside transaction or in session mode"); @@ -353,10 +355,10 @@ impl Client { match read_message(&mut self.read).await { Ok(message) => message, Err(err) => { - // Client disconnected without warning. + // Client disconnected inside a transaction. + // Clean up the server and re-use it. + // This prevents connection thrashing by bad clients. if server.in_transaction() { - // Client left dirty server. Clean up and proceed - // without thrashing this connection. server.query("ROLLBACK; DISCARD ALL;").await?; } @@ -383,13 +385,11 @@ impl Client { 'Q' => { debug!("Sending query to server"); - // TODO: implement retries here for read-only transactions. server.send(original).await?; // Read all data the server has to offer, which can be multiple messages // buffered in 8196 bytes chunks. loop { - // TODO: implement retries here for read-only transactions. let response = server.recv().await?; // Send server reply to the client. @@ -409,7 +409,6 @@ impl Client { // Report query executed statistics. self.stats.query(self.process_id, address.id); - // The transaction is over, we can release the connection back to the pool. if !server.in_transaction() { // Report transaction executed statistics. self.stats.transaction(self.process_id, address.id); @@ -429,7 +428,6 @@ impl Client { // connection before releasing into the pool. // Pgbouncer closes the connection which leads to // connection thrashing when clients misbehave. - // This pool will protect the database. :salute: if server.in_transaction() { server.query("ROLLBACK; DISCARD ALL;").await?; } @@ -468,7 +466,6 @@ impl Client { self.buffer.put(&original[..]); - // TODO: retries for read-only transactions. server.send(self.buffer.clone()).await?; self.buffer.clear(); @@ -476,7 +473,6 @@ impl Client { // Read all data the server has to offer, which can be multiple messages // buffered in 8196 bytes chunks. loop { - // TODO: retries for read-only transactions let response = server.recv().await?; match write_all_half(&mut self.write, response).await { @@ -495,11 +491,11 @@ impl Client { // Report query executed statistics. self.stats.query(self.process_id, address.id); - // Release server back to the pool if we are in transaction mode. - // If we are in session mode, we keep the server until the client disconnects. if !server.in_transaction() { self.stats.transaction(self.process_id, address.id); + // Release server back to the pool if we are in transaction mode. + // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { self.stats.server_idle(server.process_id(), address.id); break; @@ -529,11 +525,11 @@ impl Client { } }; - // Release server back to the pool if we are in transaction mode. - // If we are in session mode, we keep the server until the client disconnects. if !server.in_transaction() { self.stats.transaction(self.process_id, address.id); + // Release server back to the pool if we are in transaction mode. + // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { self.stats.server_idle(server.process_id(), address.id); break; @@ -556,7 +552,7 @@ impl Client { } } - /// Release the server from being mine. I can't cancel its queries anymore. + /// Release the server from the client: it can't cancel its queries anymore. pub fn release(&self) { let mut guard = self.client_server_map.lock(); guard.remove(&(self.process_id, self.secret_key)); @@ -565,11 +561,10 @@ impl Client { impl Drop for Client { fn drop(&mut self) { - // Disconnect the client + // Update statistics. if let Some(address_id) = self.last_address_id { self.stats.client_disconnecting(self.process_id, address_id); - // The server is now idle if let Some(process_id) = self.last_server_id { self.stats.server_idle(process_id, address_id); } diff --git a/src/config.rs b/src/config.rs index 87c40a5cc..f11aa3155 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,18 +1,20 @@ +/// Parse the configuration file. use arc_swap::{ArcSwap, Guard}; use log::{error, info}; use once_cell::sync::Lazy; use serde_derive::Deserialize; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use tokio::fs::File; use tokio::io::AsyncReadExt; use toml; -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - use crate::errors::Error; +/// Globally available configuration. static CONFIG: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Config::default())); +/// Server role: primary or replica. #[derive(Clone, PartialEq, Deserialize, Hash, std::cmp::Eq, Debug, Copy)] pub enum Role { Primary, @@ -46,6 +48,7 @@ impl PartialEq for Option { } } +/// Address identifying a PostgreSQL server uniquely. #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)] pub struct Address { pub id: usize, @@ -70,6 +73,7 @@ impl Default for Address { } impl Address { + /// Address name (aka database) used in `SHOW STATS`, `SHOW DATABASES`, and `SHOW POOLS`. pub fn name(&self) -> String { match self.role { Role::Primary => format!("shard_{}_primary", self.shard), @@ -79,6 +83,7 @@ impl Address { } } +/// PostgreSQL user. #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Deserialize, Debug)] pub struct User { pub name: String, @@ -94,6 +99,7 @@ impl Default for User { } } +/// General configuration. #[derive(Deserialize, Debug, Clone)] pub struct General { pub host: String, @@ -119,6 +125,7 @@ impl Default for General { } } +/// Shard configuration. #[derive(Deserialize, Debug, Clone)] pub struct Shard { pub servers: Vec<(String, u16, String)>, @@ -134,6 +141,7 @@ impl Default for Shard { } } +/// Query Router configuration. #[derive(Deserialize, Debug, Clone)] pub struct QueryRouter { pub default_role: String, @@ -153,6 +161,7 @@ impl Default for QueryRouter { } } +/// Configuration wrapper. #[derive(Deserialize, Debug, Clone)] pub struct Config { pub path: Option, @@ -217,6 +226,7 @@ impl From<&Config> for std::collections::HashMap { } impl Config { + /// Print current configuration. pub fn show(&self) { info!("Pool size: {}", self.general.pool_size); info!("Pool mode: {}", self.general.pool_mode); @@ -231,11 +241,14 @@ impl Config { } } +/// Get a read-only instance of the configuration +/// from anywhere in the app. +/// ArcSwap makes this cheap and quick. pub fn get_config() -> Guard> { CONFIG.load() } -/// Parse the config. +/// Parse the configuration file located at the path. pub async fn parse(path: &str) -> Result<(), Error> { let mut contents = String::new(); let mut file = match File::open(path).await { @@ -346,6 +359,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { config.path = Some(path.to_string()); + // Update the configuration globally. CONFIG.store(Arc::new(config.clone())); Ok(()) diff --git a/src/constants.rs b/src/constants.rs index 074811e1d..a0b63c09c 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -1,7 +1,6 @@ /// Various protocol constants, as defined in -/// https://www.postgresql.org/docs/12/protocol-message-formats.html +/// /// and elsewhere in the source code. -/// Also other constants we use elsewhere. // Used in the StartupMessage to indicate regular handshake. pub const PROTOCOL_VERSION_NUMBER: i32 = 196608; diff --git a/src/errors.rs b/src/errors.rs index 1fc26bbf9..b42d32197 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,12 +1,12 @@ +/// Errors. + +/// Various errors. #[derive(Debug, PartialEq)] pub enum Error { SocketError, - // ClientDisconnected, ClientBadStartup, ProtocolSyncError, ServerError, - // ServerTimeout, - // DirtyServer, BadConfig, AllServersDown, } diff --git a/src/main.rs b/src/main.rs index 97b198880..c7a82e179 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,19 +58,15 @@ mod server; mod sharding; mod stats; -// Support for query cancellation: this maps our process_ids and -// secret keys to the backend's. use config::get_config; use pool::{ClientServerMap, ConnectionPool}; use stats::{Collector, Reporter}; -/// Main! #[tokio::main(worker_threads = 4)] async fn main() { env_logger::init(); info!("Welcome to PgCat! Meow."); - // Prepare regexes if !query_router::QueryRouter::setup() { error!("Could not setup query router"); return; @@ -84,7 +80,6 @@ async fn main() { String::from("pgcat.toml") }; - // Prepare the config match config::parse(&config_file).await { Ok(_) => (), Err(err) => { @@ -94,8 +89,8 @@ async fn main() { }; let config = get_config(); - let addr = format!("{}:{}", config.general.host, config.general.port); + let listener = match TcpListener::bind(&addr).await { Ok(sock) => sock, Err(err) => { @@ -105,18 +100,20 @@ async fn main() { }; info!("Running on {}", addr); + config.show(); // Tracks which client is connected to which server for query cancellation. let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); - // Collect statistics and send them to StatsD + // Statistics reporting. let (tx, rx) = mpsc::channel(100); - // Connection pool for all shards and replicas + // Connection pool that allows to query all shards and replicas. let mut pool = ConnectionPool::from_config(client_server_map.clone(), Reporter::new(tx.clone())).await; + // Statistics collector task. let collector_tx = tx.clone(); let addresses = pool.databases(); tokio::task::spawn(async move { @@ -135,7 +132,7 @@ async fn main() { info!("Waiting for clients"); - // Main app runs here. + // Client connection loop. tokio::task::spawn(async move { loop { let pool = pool.clone(); @@ -151,7 +148,7 @@ async fn main() { } }; - // Client goes to another thread, bye. + // Handle client. tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); match client::Client::startup(socket, client_server_map, server_info, reporter) @@ -185,7 +182,7 @@ async fn main() { } }); - // Reload config + // Reload config: // kill -SIGHUP $(pgrep pgcat) tokio::task::spawn(async move { let mut stream = unix_signal(SignalKind::hangup()).unwrap(); @@ -205,6 +202,7 @@ async fn main() { } }); + // Exit on Ctrl-C (SIGINT) and SIGTERM. let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); tokio::select! { diff --git a/src/messages.rs b/src/messages.rs index 473c8de6b..3420bb4ff 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -222,7 +222,7 @@ pub async fn custom_protocol_response_ok( /// Send a custom error message to the client. /// Tell the client we are ready for the next query and no rollback is necessary. -/// Docs on error codes: https://www.postgresql.org/docs/12/errcodes-appendix.html +/// Docs on error codes: . pub async fn error_response(stream: &mut OwnedWriteHalf, message: &str) -> Result<(), Error> { let mut error = BytesMut::new(); @@ -339,6 +339,7 @@ pub fn row_description(columns: &Vec<(&str, DataType)>) -> BytesMut { res } +/// Create a DataRow message. pub fn data_row(row: &Vec) -> BytesMut { let mut res = BytesMut::new(); let mut data_row = BytesMut::new(); @@ -358,6 +359,7 @@ pub fn data_row(row: &Vec) -> BytesMut { res } +/// Create a CommandComplete message. pub fn command_complete(command: &str) -> BytesMut { let cmd = BytesMut::from(format!("{}\0", command).as_bytes()); let mut res = BytesMut::new(); diff --git a/src/pool.rs b/src/pool.rs index 046a6811a..0da82786c 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,24 +1,23 @@ -/// Pooling and failover and banlist. +/// Pooling, failover and banlist. use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection}; use bytes::BytesMut; use chrono::naive::NaiveDateTime; use log::{debug, error, info, warn}; use parking_lot::{Mutex, RwLock}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Instant; use crate::config::{get_config, Address, Role, User}; use crate::errors::Error; use crate::server::Server; use crate::stats::Reporter; -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Instant; - -// Banlist: bad servers go in here. pub type BanList = Arc>>>; pub type ClientServerMap = Arc>>; +/// The globally accessible connection pool. #[derive(Clone, Debug)] pub struct ConnectionPool { databases: Vec>>, @@ -29,7 +28,7 @@ pub struct ConnectionPool { } impl ConnectionPool { - /// Construct the connection pool from a config file. + /// Construct the connection pool from the configuration. pub async fn from_config( client_server_map: ClientServerMap, stats: Reporter, @@ -204,10 +203,9 @@ impl ConnectionPool { } while allowed_attempts > 0 { - // Round-robin each client's queries. - // If a client only sends one query and then disconnects, it doesn't matter - // which replica it'll go to. + // Round-robin replicas. self.round_robin += 1; + let index = self.round_robin % addresses.len(); let address = &addresses[index]; @@ -239,7 +237,7 @@ impl ConnectionPool { } }; - // // Check if this server is alive with a health check + // // Check if this server is alive with a health check. let server = &mut *conn; let healthcheck_timeout = get_config().general.healthcheck_timeout; @@ -251,7 +249,7 @@ impl ConnectionPool { ) .await { - // Check if health check succeeded + // Check if health check succeeded. Ok(res) => match res { Ok(_) => { self.stats @@ -259,8 +257,11 @@ impl ConnectionPool { self.stats.server_idle(conn.process_id(), address.id); return Ok((conn, address.clone())); } + + // Health check failed. Err(_) => { error!("Banning replica {} because of failed health check", index); + // Don't leave a bad connection in the pool. server.mark_bad(); @@ -271,7 +272,8 @@ impl ConnectionPool { continue; } }, - // Health check never came back, database is really really down + + // Health check timed out. Err(_) => { error!("Banning replica {} because of health check timeout", index); // Don't leave a bad connection in the pool. @@ -358,14 +360,18 @@ impl ConnectionPool { } } + /// Get the number of configured shards. pub fn shards(&self) -> usize { self.databases.len() } + /// Get the number of servers (primary and replicas) + /// configured for a shard. pub fn servers(&self, shard: usize) -> usize { self.addresses[shard].len() } + /// Get the total number of servers (databases) we are connected to. pub fn databases(&self) -> usize { let mut databases = 0; for shard in 0..self.shards() { @@ -374,15 +380,18 @@ impl ConnectionPool { databases } + /// Get pool state for a particular shard server as reported by bb8. pub fn pool_state(&self, shard: usize, server: usize) -> bb8::State { self.databases[shard][server].state() } + /// Get the address information for a shard server. pub fn address(&self, shard: usize, server: usize) -> &Address { &self.addresses[shard][server] } } +/// Wrapper for the bb8 connection pool. pub struct ServerPool { address: Address, user: User, @@ -427,6 +436,7 @@ impl ManageConnection for ServerPool { let process_id = rand::random::(); self.stats.server_login(process_id, self.address.id); + // Connect to the PostgreSQL server. match Server::startup( &self.address, &self.user, diff --git a/src/query_router.rs b/src/query_router.rs index daa917142..1f20079d3 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1,5 +1,3 @@ -use crate::config::{get_config, Role}; -use crate::sharding::{Sharder, ShardingFunction}; /// Route queries automatically based on explicitely requested /// or implied query characteristics. use bytes::{Buf, BytesMut}; @@ -10,6 +8,10 @@ use sqlparser::ast::Statement::{Query, StartTransaction}; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; +use crate::config::{get_config, Role}; +use crate::sharding::{Sharder, ShardingFunction}; + +/// Regexes used to parse custom commands. const CUSTOM_SQL_REGEXES: [&str; 5] = [ r"(?i)^ *SET SHARDING KEY TO '?([0-9]+)'? *;? *$", r"(?i)^ *SET SHARD TO '?([0-9]+|ANY)'? *;? *$", @@ -18,6 +20,7 @@ const CUSTOM_SQL_REGEXES: [&str; 5] = [ r"(?i)^ *SHOW SERVER ROLE *;? *$", ]; +/// Custom commands. #[derive(PartialEq, Debug)] pub enum Command { SetShardingKey, @@ -27,37 +30,39 @@ pub enum Command { ShowServerRole, } -// Quick test +/// Quickly test for match when a query is received. static CUSTOM_SQL_REGEX_SET: OnceCell = OnceCell::new(); -// Capture value +// Get the value inside the custom command. static CUSTOM_SQL_REGEX_LIST: OnceCell> = OnceCell::new(); +/// The query router. pub struct QueryRouter { - // By default, queries go here, unless we have better information - // about what the client wants. + /// By default, queries go here, unless we have better information + /// about what the client wants. default_server_role: Option, - // Number of shards in the cluster. + /// Number of shards in the cluster. shards: usize, - // Which shard we should be talking to right now. + /// Which shard we should be talking to right now. active_shard: Option, - // Should we be talking to a primary or a replica? + /// Which server should we be talking to. active_role: Option, - // Include the primary into the replica pool? + /// Include the primary into the replica pool for reads. primary_reads_enabled: bool, - // Should we try to parse queries? + /// Should we try to parse queries to route them to replicas or primary automatically. query_parser_enabled: bool, - // Which sharding function are we using? + /// Which sharding function we're using. sharding_function: ShardingFunction, } impl QueryRouter { + /// One-time initialization of regexes. pub fn setup() -> bool { let set = match RegexSet::new(&CUSTOM_SQL_REGEXES) { Ok(rgx) => rgx, @@ -88,6 +93,7 @@ impl QueryRouter { } } + /// Create a new instance of the query router. Each client gets its own. pub fn new() -> QueryRouter { let config = get_config(); @@ -120,6 +126,7 @@ impl QueryRouter { pub fn try_execute_command(&mut self, mut buf: BytesMut) -> Option<(Command, String)> { let code = buf.get_u8() as char; + // Only simple protocol supported for commands. if code != 'Q' { return None; } @@ -158,8 +165,7 @@ impl QueryRouter { // figured out a better way just yet. I think I can write a single Regex // that matches all 5 custom SQL patterns, but maybe that's not very legible? // - // I think this is faster than running the Regex engine 5 times, so - // this is a strong maybe for me so far. + // I think this is faster than running the Regex engine 5 times. match regex_list[matches[0]].captures(&query) { Some(captures) => match captures.get(1) { Some(value) => value.as_str().to_string(), @@ -221,7 +227,6 @@ impl QueryRouter { } "default" => { - // TODO: reset query parser to default here. self.active_role = self.default_server_role; self.query_parser_enabled = get_config().query_router.query_parser_enabled; self.active_role @@ -243,12 +248,14 @@ impl QueryRouter { let len = buf.get_i32() as usize; let query = match code { + // Query 'Q' => { let query = String::from_utf8_lossy(&buf[..len - 5]).to_string(); debug!("Query: '{}'", query); query } + // Parse (prepared statement) 'P' => { let mut start = 0; let mut end; @@ -271,6 +278,7 @@ impl QueryRouter { query.replace("$", "") // Remove placeholders turning them into "values" } + _ => return false, }; @@ -334,6 +342,7 @@ impl QueryRouter { self.query_parser_enabled } + /// Allows to toggle primary reads in tests. #[allow(dead_code)] pub fn toggle_primary_reads(&mut self, value: bool) { self.primary_reads_enabled = value; diff --git a/src/server.rs b/src/server.rs index aaab47c0c..91858275a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,6 +1,6 @@ +/// Implementation of the PostgreSQL server (database) protocol. +/// Here we are pretending to the a Postgres client. use bytes::{Buf, BufMut, BytesMut}; -///! Implementation of the PostgreSQL server (database) protocol. -///! Here we are pretending to the a Postgres client. use log::{debug, error, info, trace}; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ @@ -17,42 +17,42 @@ use crate::ClientServerMap; /// Server state. pub struct Server { - // Server host, e.g. localhost, - // port, e.g. 5432, and role, e.g. primary or replica. + /// Server host, e.g. localhost, + /// port, e.g. 5432, and role, e.g. primary or replica. address: Address, - // Buffered read socket. + /// Buffered read socket. read: BufReader, - // Unbuffered write socket (our client code buffers). + /// Unbuffered write socket (our client code buffers). write: OwnedWriteHalf, - // Our server response buffer. We buffer data before we give it to the client. + /// Our server response buffer. We buffer data before we give it to the client. buffer: BytesMut, - // Server information the server sent us over on startup. + /// Server information the server sent us over on startup. server_info: BytesMut, - // Backend id and secret key used for query cancellation. + /// Backend id and secret key used for query cancellation. process_id: i32, secret_key: i32, - // Is the server inside a transaction or idle. + /// Is the server inside a transaction or idle. in_transaction: bool, - // Is there more data for the client to read. + /// Is there more data for the client to read. data_available: bool, - // Is the server broken? We'll remote it from the pool if so. + /// Is the server broken? We'll remote it from the pool if so. bad: bool, - // Mapping of clients and servers used for query cancellation. + /// Mapping of clients and servers used for query cancellation. client_server_map: ClientServerMap, - // Server connected at. + /// Server connected at. connected_at: chrono::naive::NaiveDateTime, - // Reports various metrics, e.g. data sent & received. + /// Reports various metrics, e.g. data sent & received. stats: Reporter, } @@ -77,7 +77,7 @@ impl Server { trace!("Sending StartupMessage"); - // Send the startup packet telling the server we're a normal Postgres client. + // StartupMessage startup(&mut stream, &user.name, database).await?; let mut server_info = BytesMut::new(); @@ -187,7 +187,7 @@ impl Server { // BackendKeyData 'K' => { // The frontend must save these values if it wishes to be able to issue CancelRequest messages later. - // See: https://www.postgresql.org/docs/12/protocol-message-formats.html + // See: . process_id = match stream.read_i32().await { Ok(id) => id, Err(_) => return Err(Error::SocketError), @@ -208,8 +208,6 @@ impl Server { Err(_) => return Err(Error::SocketError), }; - // This is the last step in the client-server connection setup, - // and indicates the server is ready for to query it. let (read, write) = stream.into_split(); return Ok(Server { @@ -342,8 +340,7 @@ impl Server { // More data is available after this message, this is not the end of the reply. self.data_available = true; - // Don't flush yet, the more we buffer, the faster this goes... - // up to a limit of course. + // Don't flush yet, the more we buffer, the faster this goes...up to a limit. if self.buffer.len() >= 8196 { break; } @@ -411,7 +408,7 @@ impl Server { /// Indicate that this server connection cannot be re-used and must be discarded. pub fn mark_bad(&mut self) { - error!("Server marked bad"); + error!("Server {:?} marked bad", self.address); self.bad = true; } @@ -462,6 +459,7 @@ impl Server { self.address.clone() } + /// Get the server's unique identifier. pub fn process_id(&self) -> i32 { self.process_id } @@ -481,9 +479,10 @@ impl Drop for Server { match self.write.try_write(&bytes) { Ok(_) => (), - Err(_) => (), + Err(_) => debug!("Dirty shutdown"), }; + // Should not matter. self.bad = true; let now = chrono::offset::Utc::now().naive_utc(); diff --git a/src/sharding.rs b/src/sharding.rs index 23eccc142..c332c6012 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -1,20 +1,27 @@ +/// Implements various sharding functions. use sha1::{Digest, Sha1}; -// https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/include/catalog/partition.h#L20 +/// See: . const PARTITION_HASH_SEED: u64 = 0x7A5B22367996DCFD; +/// The sharding functions we support. #[derive(Debug, PartialEq, Copy, Clone)] pub enum ShardingFunction { PgBigintHash, Sha1, } +/// The sharder. pub struct Sharder { + /// Number of shards in the cluster. shards: usize, + + /// The sharding function in use. sharding_function: ShardingFunction, } impl Sharder { + /// Create new instance of the sharder. pub fn new(shards: usize, sharding_function: ShardingFunction) -> Sharder { Sharder { shards, @@ -22,6 +29,7 @@ impl Sharder { } } + /// Compute the shard given sharding key. pub fn shard(&self, key: i64) -> usize { match self.sharding_function { ShardingFunction::PgBigintHash => self.pg_bigint_hash(key), @@ -31,7 +39,7 @@ impl Sharder { /// Hash function used by Postgres to determine which partition /// to put the row in when using HASH(column) partitioning. - /// Source: https://github.com/postgres/postgres/blob/27b77ecf9f4d5be211900eda54d8155ada50d696/src/common/hashfn.c#L631 + /// Source: . /// Supports only 1 bigint at the moment, but we can add more later. fn pg_bigint_hash(&self, key: i64) -> usize { let mut lohalf = key as u32; @@ -119,6 +127,7 @@ impl Sharder { a } + #[inline] fn pg_u32_hash(k: u32) -> u64 { let mut a: u32 = 0x9e3779b9 as u32 + std::mem::size_of::() as u32 + 3923095 as u32; let mut b = a; diff --git a/src/stats.rs b/src/stats.rs index 1b7ccda21..e44578d98 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -5,12 +5,12 @@ use parking_lot::Mutex; use std::collections::HashMap; use tokio::sync::mpsc::{Receiver, Sender}; -// Latest stats updated every second; used in SHOW STATS and other admin commands. +/// Latest stats updated every second; used in SHOW STATS and other admin commands. static LATEST_STATS: Lazy>>> = Lazy::new(|| Mutex::new(HashMap::new())); -// Statistics period used for average calculations. -// 15 seconds. +/// Statistics period used for average calculations. +/// 15 seconds. static STAT_PERIOD: u64 = 15000; /// The names for the events reported From 9513cdcfcccd762bca8acd3abf71e12d66d50ec7 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 9 May 2022 11:09:22 -0500 Subject: [PATCH 132/551] Poorly behaved client fix (#65) * Poorly behaved client fix * yes officer * fix tests * no useless rescue * Looks ok --- src/client.rs | 6 ++++-- tests/ruby/Gemfile | 1 + tests/ruby/Gemfile.lock | 21 +++++++++++++++++++++ tests/ruby/tests.rb | 17 +++++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/client.rs b/src/client.rs index 87f621562..976121b96 100644 --- a/src/client.rs +++ b/src/client.rs @@ -359,7 +359,8 @@ impl Client { // Clean up the server and re-use it. // This prevents connection thrashing by bad clients. if server.in_transaction() { - server.query("ROLLBACK; DISCARD ALL;").await?; + server.query("ROLLBACK").await?; + server.query("DISCARD ALL").await?; } return Err(err); @@ -429,7 +430,8 @@ impl Client { // Pgbouncer closes the connection which leads to // connection thrashing when clients misbehave. if server.in_transaction() { - server.query("ROLLBACK; DISCARD ALL;").await?; + server.query("ROLLBACK").await?; + server.query("DISCARD ALL").await?; } return Ok(()); diff --git a/tests/ruby/Gemfile b/tests/ruby/Gemfile index 94fb8c3d3..05684c985 100644 --- a/tests/ruby/Gemfile +++ b/tests/ruby/Gemfile @@ -2,3 +2,4 @@ source "https://rubygems.org" gem "pg" gem "activerecord" +gem "rubocop" diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock index 7b7d44841..045e2f61a 100644 --- a/tests/ruby/Gemfile.lock +++ b/tests/ruby/Gemfile.lock @@ -11,13 +11,33 @@ GEM i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) + ast (2.4.2) concurrent-ruby (1.1.9) i18n (1.10.0) concurrent-ruby (~> 1.0) minitest (5.15.0) + parallel (1.22.1) + parser (3.1.2.0) + ast (~> 2.4.1) pg (1.3.2) + rainbow (3.1.1) + regexp_parser (2.3.1) + rexml (3.2.5) + rubocop (1.29.0) + parallel (~> 1.10) + parser (>= 3.1.0.0) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.17.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.17.0) + parser (>= 3.1.1.0) + ruby-progressbar (1.11.0) tzinfo (2.0.4) concurrent-ruby (~> 1.0) + unicode-display_width (2.1.0) PLATFORMS x86_64-linux @@ -25,6 +45,7 @@ PLATFORMS DEPENDENCIES activerecord pg + rubocop BUNDLED WITH 2.3.7 diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index 622ac0146..5b032a985 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -1,6 +1,9 @@ # frozen_string_literal: true require 'active_record' +require 'pg' + +$stdout.sync = true # Uncomment these two to see all queries. # ActiveRecord.verbose_query_logs = true @@ -110,3 +113,17 @@ def down rescue ActiveRecord::StatementInvalid puts 'OK' end + +# Test evil clients +def poorly_behaved_client + conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/rails_dev") + conn.async_exec 'BEGIN' + conn.async_exec 'SELECT 1' + + conn.close + puts 'Bad client ok' +end + +25.times do + poorly_behaved_client +end From b97b14d52d2aa49a3e3aecfb70ff9f6dd4096cab Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 14 May 2022 21:35:33 -0700 Subject: [PATCH 133/551] Possible fix for clients waiting stat leak (#68) --- src/pool.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pool.rs b/src/pool.rs index 0da82786c..cc8b6883d 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -215,12 +215,14 @@ impl ConnectionPool { // as per request. If no specific role is requested, the first // available will be chosen. if address.role != role { + self.stats.client_disconnecting(process_id, address.id); continue; } allowed_attempts -= 1; if self.is_banned(address, shard, role) { + self.stats.client_disconnecting(process_id, address.id); continue; } From 6dc8af1597e55d3fc00d1011f2ce85777c2c8f8e Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 17 May 2022 13:05:25 -0700 Subject: [PATCH 134/551] Reduce traffic on the stats channel (#69) --- src/pool.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/pool.rs b/src/pool.rs index cc8b6883d..53803e7fe 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -209,23 +209,22 @@ impl ConnectionPool { let index = self.round_robin % addresses.len(); let address = &addresses[index]; - self.stats.client_waiting(process_id, address.id); - // Make sure you're getting a primary or a replica // as per request. If no specific role is requested, the first // available will be chosen. if address.role != role { - self.stats.client_disconnecting(process_id, address.id); continue; } allowed_attempts -= 1; if self.is_banned(address, shard, role) { - self.stats.client_disconnecting(process_id, address.id); continue; } + // Indicate we're waiting on a server connection from a pool. + self.stats.client_waiting(process_id, address.id); + // Check if we can connect let mut conn = match self.databases[shard][index].get().await { Ok(conn) => conn, From 85f15c0e069615f3db159d5217433433506d1316 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 18 May 2022 16:24:03 -0700 Subject: [PATCH 135/551] Fix memory leak in client/server mapping (#71) --- src/client.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client.rs b/src/client.rs index 976121b96..047a3ac62 100644 --- a/src/client.rs +++ b/src/client.rs @@ -434,6 +434,8 @@ impl Client { server.query("DISCARD ALL").await?; } + self.release(); + return Ok(()); } From 70ede7db6026afb584f62399c56b7d706551ee8f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 3 Jun 2022 00:15:50 -0700 Subject: [PATCH 136/551] Pass application_name to server (#73) * Pass application_name to server * fmt --- src/client.rs | 8 ++++++++ tests/ruby/tests.rb | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index 047a3ac62..45934fd6d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -342,6 +342,14 @@ impl Client { server.address() ); + // Set application_name if any. + // TODO: investigate other parameters and set them too. + if self.parameters.contains_key("application_name") { + server + .set_name(&self.parameters["application_name"]) + .await?; + } + // Transaction loop. Multiple queries can be issued by the client here. // The connection belongs to the client until the transaction is over, // or until the client disconnects if we are in session mode. diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index 5b032a985..983619f0f 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -16,6 +16,7 @@ username: 'sharding_user', password: 'sharding_user', database: 'rails_dev', + application_name: 'testing_pgcat', prepared_statements: false, # Transaction mode advisory_locks: false # Same ) @@ -116,7 +117,7 @@ def down # Test evil clients def poorly_behaved_client - conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/rails_dev") + conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/rails_dev?application_name=testing_pgcat") conn.async_exec 'BEGIN' conn.async_exec 'SELECT 1' From e12751dacafab70efa4b4188ded805b09bb249f9 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 5 Jun 2022 09:48:06 -0700 Subject: [PATCH 137/551] Only set application_name if it's different (#74) * Only set application_name if it's different * keep server named pgcat until something else changes --- src/client.rs | 2 ++ src/server.rs | 23 ++++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/client.rs b/src/client.rs index 45934fd6d..2c96ccc2d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -369,6 +369,7 @@ impl Client { if server.in_transaction() { server.query("ROLLBACK").await?; server.query("DISCARD ALL").await?; + server.set_name("pgcat").await?; } return Err(err); @@ -440,6 +441,7 @@ impl Client { if server.in_transaction() { server.query("ROLLBACK").await?; server.query("DISCARD ALL").await?; + server.set_name("pgcat").await?; } self.release(); diff --git a/src/server.rs b/src/server.rs index 91858275a..0dd051e59 100644 --- a/src/server.rs +++ b/src/server.rs @@ -54,6 +54,9 @@ pub struct Server { /// Reports various metrics, e.g. data sent & received. stats: Reporter, + + /// Application name using the server at the moment. + application_name: String, } impl Server { @@ -210,7 +213,7 @@ impl Server { let (read, write) = stream.into_split(); - return Ok(Server { + let mut server = Server { address: address.clone(), read: BufReader::new(read), write: write, @@ -224,7 +227,12 @@ impl Server { client_server_map: client_server_map, connected_at: chrono::offset::Utc::now().naive_utc(), stats: stats, - }); + application_name: String::new(), + }; + + server.set_name("pgcat").await?; + + return Ok(server); } // We have an unexpected message from the server during this exchange. @@ -448,9 +456,14 @@ impl Server { /// A shorthand for `SET application_name = $1`. #[allow(dead_code)] pub async fn set_name(&mut self, name: &str) -> Result<(), Error> { - Ok(self - .query(&format!("SET application_name = '{}'", name)) - .await?) + if self.application_name != name { + self.application_name = name.to_string(); + Ok(self + .query(&format!("SET application_name = '{}'", name)) + .await?) + } else { + Ok(()) + } } /// Get the servers address. From 5d62dbc087d6b44bb94af45eea9b0f5997a8c581 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 19:59:50 -0700 Subject: [PATCH 138/551] Bump regex from 1.5.4 to 1.5.5 (#75) Bumps [regex](https://github.com/rust-lang/regex) from 1.5.4 to 1.5.5. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.5.4...1.5.5) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe3c2999b..19df07b00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -462,9 +462,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" dependencies = [ "aho-corasick", "memchr", From 61908038b8cf11a9c603f3006c3d2670d52f0398 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 18 Jun 2022 18:36:00 -0700 Subject: [PATCH 139/551] Implement SCRAM-SHA-256 for server authentication (PG14) (#76) * Implement SCRAM-SHA-256 * test it * trace * move to community for auth * hmm --- .circleci/config.yml | 10 +- .circleci/run_tests.sh | 4 +- Cargo.lock | 87 +++++++++++- Cargo.toml | 6 +- src/constants.rs | 7 + src/main.rs | 1 + src/scram.rs | 311 +++++++++++++++++++++++++++++++++++++++++ src/server.rs | 80 +++++++++++ 8 files changed, 494 insertions(+), 12 deletions(-) create mode 100644 src/scram.rs diff --git a/.circleci/config.yml b/.circleci/config.yml index 63bfee52a..09fc893ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,13 +12,15 @@ jobs: - image: cimg/rust:1.58.1 environment: RUST_LOG: info - - image: cimg/postgres:14.0 - auth: - username: mydockerhub-user - password: $DOCKERHUB_PASSWORD + - image: postgres:14 + # auth: + # username: mydockerhub-user + # password: $DOCKERHUB_PASSWORD environment: POSTGRES_USER: postgres POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: scram-sha-256 # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 9dbca9d20..f325d1665 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -12,7 +12,7 @@ function start_pgcat() { } # Setup the database with shards and user -psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql +PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard0 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard2 -i @@ -72,7 +72,7 @@ psql -h 127.0.0.1 -p 6432 -d pgbouncer -c "SET client_encoding TO 'utf8'" > /dev (! psql -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) # Start PgCat in debug to demonstrate failover better -start_pgcat "debug" +start_pgcat "trace" # Add latency to the replica at port 5433 slightly above the healthcheck timeout toxiproxy-cli toxic add -t latency -a latency=300 postgres_replica diff --git a/Cargo.lock b/Cargo.lock index 19df07b00..51bbc6e58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bb8" version = "0.7.1" @@ -109,22 +115,23 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d6b536309245c849479fba3da410962a43ed8e51c26b729208ec0ac2798d0" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" dependencies = [ "generic-array", + "typenum", ] [[package]] name = "digest" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b697d66081d42af4fba142d56918a3cb21dc8eb63372c6b85d14f44fb9c5979b" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ "block-buffer", "crypto-common", - "generic-array", + "subtle", ] [[package]] @@ -205,6 +212,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "humantime" version = "2.1.0" @@ -356,10 +372,12 @@ version = "0.1.0-beta2" dependencies = [ "arc-swap", "async-trait", + "base64", "bb8", "bytes", "chrono", "env_logger", + "hmac", "log", "md-5", "num_cpus", @@ -370,7 +388,9 @@ dependencies = [ "serde", "serde_derive", "sha-1", + "sha2", "sqlparser", + "stringprep", "tokio", "toml", ] @@ -511,6 +531,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -541,6 +572,22 @@ dependencies = [ "log", ] +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "syn" version = "1.0.86" @@ -572,6 +619,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "tokio" version = "1.16.1" @@ -617,6 +679,21 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-xid" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index d41657ad1..ae18d7b73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "0.1.0-beta2" +version = "0.2.0-beta1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -25,3 +25,7 @@ log = "0.4" arc-swap = "1" env_logger = "0.9" parking_lot = "0.11" +hmac = "0.12" +sha2 = "0.10" +base64 = "0.13" +stringprep = "0.1" \ No newline at end of file diff --git a/src/constants.rs b/src/constants.rs index a0b63c09c..0900d7cce 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -14,6 +14,13 @@ pub const CANCEL_REQUEST_CODE: i32 = 80877102; // AuthenticationMD5Password pub const MD5_ENCRYPTED_PASSWORD: i32 = 5; +// SASL +pub const SASL: i32 = 10; +pub const SASL_CONTINUE: i32 = 11; +pub const SASL_FINAL: i32 = 12; +pub const SCRAM_SHA_256: &str = "SCRAM-SHA-256"; +pub const NONCE_LENGTH: usize = 24; + // AuthenticationOk pub const AUTHENTICATION_SUCCESSFUL: i32 = 0; diff --git a/src/main.rs b/src/main.rs index c7a82e179..c22391edd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -54,6 +54,7 @@ mod errors; mod messages; mod pool; mod query_router; +mod scram; mod server; mod sharding; mod stats; diff --git a/src/scram.rs b/src/scram.rs new file mode 100644 index 000000000..58096fa8e --- /dev/null +++ b/src/scram.rs @@ -0,0 +1,311 @@ +// SCRAM authentication...largely copy/pasted from +// https://github.com/sfackler/rust-postgres/. + +use bytes::BytesMut; +use hmac::{Hmac, Mac}; +use rand::{self, Rng}; +use sha2::digest::FixedOutput; +use sha2::{Digest, Sha256}; + +use std::fmt::Write; + +use crate::constants::*; +use crate::errors::Error; + +fn normalize(pass: &[u8]) -> Vec { + let pass = match std::str::from_utf8(pass) { + Ok(pass) => pass, + Err(_) => return pass.to_vec(), + }; + + match stringprep::saslprep(pass) { + Ok(pass) => pass.into_owned().into_bytes(), + Err(_) => pass.as_bytes().to_vec(), + } +} + +pub struct ScramSha256 { + password: String, + salted_password: [u8; 32], + auth_message: String, + message: BytesMut, + nonce: String, +} + +impl ScramSha256 { + pub fn new(password: &str) -> ScramSha256 { + let mut rng = rand::thread_rng(); + let nonce = (0..NONCE_LENGTH) + .map(|_| { + let mut v = rng.gen_range(0x21u8..0x7e); + if v == 0x2c { + v = 0x7e + } + v as char + }) + .collect::(); + + Self::from_nonce(password, &nonce) + } + + pub fn from_nonce(password: &str, nonce: &str) -> ScramSha256 { + let message = BytesMut::from(&format!("{}n=,r={}", "n,,", nonce).as_bytes()[..]); + + ScramSha256 { + password: password.to_string(), + nonce: String::from(nonce), + message, + salted_password: [0u8; 32], + auth_message: String::new(), + } + } + + pub fn message(&mut self) -> BytesMut { + self.message.clone() + } + + pub fn update(&mut self, message: &BytesMut) -> Result { + let server_message = Message::parse(message)?; + + if !server_message.nonce.starts_with(&self.nonce) { + // trace!("Bad server nonce"); + return Err(Error::ProtocolSyncError); + } + + let salt = match base64::decode(&server_message.salt) { + Ok(salt) => salt, + Err(_) => return Err(Error::ProtocolSyncError), + }; + + let salted_password = Self::hi( + &normalize(&self.password.as_bytes()[..]), + &salt, + server_message.iterations, + ); + self.salted_password = salted_password; + + let mut hmac = Hmac::::new_from_slice(&salted_password) + .expect("HMAC is able to accept all key sizes"); + hmac.update(b"Client Key"); + let client_key = hmac.finalize().into_bytes(); + + let mut hash = Sha256::default(); + hash.update(client_key.as_slice()); + let stored_key = hash.finalize_fixed(); + + let mut cbind_input = vec![]; + cbind_input.extend("n,,".as_bytes()); + let cbind_input = base64::encode(&cbind_input); + + self.message.clear(); + write!( + &mut self.message, + "c={},r={}", + cbind_input, server_message.nonce + ) + .unwrap(); + + let auth_message = format!( + "n=,r={},{},{}", + self.nonce, + String::from_utf8_lossy(&message[..]), + String::from_utf8_lossy(&self.message[..]) + ); + + let mut hmac = Hmac::::new_from_slice(&stored_key) + .expect("HMAC is able to accept all key sizes"); + hmac.update(auth_message.as_bytes()); + let client_signature = hmac.finalize().into_bytes(); + + let mut client_proof = client_key; + for (proof, signature) in client_proof.iter_mut().zip(client_signature) { + *proof ^= signature; + } + + write!(&mut self.message, ",p={}", base64::encode(&*client_proof)).unwrap(); + + self.auth_message = auth_message; + + Ok(self.message.clone()) + } + + pub fn finish(&mut self, message: &BytesMut) -> Result<(), Error> { + let final_message = FinalMessage::parse(message)?; + + let verifier = match base64::decode(&final_message.value) { + Ok(verifier) => verifier, + Err(_) => return Err(Error::ProtocolSyncError), + }; + + let mut hmac = Hmac::::new_from_slice(&self.salted_password) + .expect("HMAC is able to accept all key sizes"); + hmac.update(b"Server Key"); + let server_key = hmac.finalize().into_bytes(); + + let mut hmac = Hmac::::new_from_slice(&server_key) + .expect("HMAC is able to accept all key sizes"); + hmac.update(self.auth_message.as_bytes()); + + match hmac.verify_slice(&verifier) { + Ok(_) => Ok(()), + Err(_) => return Err(Error::ServerError), + } + } + + // https://github.com/sfackler/rust-postgres/blob/c3a029e60c1c0bd0be947049859b8fa5bd5ac220/postgres-protocol/src/authentication/sasl.rs#L35 + fn hi(str: &[u8], salt: &[u8], i: u32) -> [u8; 32] { + let mut hmac = + Hmac::::new_from_slice(str).expect("HMAC is able to accept all key sizes"); + hmac.update(salt); + hmac.update(&[0, 0, 0, 1]); + let mut prev = hmac.finalize().into_bytes(); + + let mut hi = prev; + + for _ in 1..i { + let mut hmac = Hmac::::new_from_slice(str).expect("already checked above"); + hmac.update(&prev); + prev = hmac.finalize().into_bytes(); + + for (hi, prev) in hi.iter_mut().zip(prev) { + *hi ^= prev; + } + } + + hi.into() + } +} + +#[derive(Default, Debug)] +struct Message { + nonce: String, + salt: String, + iterations: u32, +} + +impl Message { + fn parse(message: &BytesMut) -> Result { + if !message.starts_with(b"r=") { + return Err(Error::ProtocolSyncError); + } + + let mut i = 2; + + while message[i] != b',' && i < message.len() { + i += 1; + } + + let nonce = String::from_utf8_lossy(&message[2..i]).to_string(); + + // Skip the , + i += 1; + + if !&message[i..].starts_with(b"s=") { + return Err(Error::ProtocolSyncError); + } + + // Skip the s= + i += 2; + + let s = i; + while message[i] != b',' && i < message.len() { + i += 1; + } + + let salt = String::from_utf8_lossy(&message[s..i]).to_string(); + + // Skip the , + i += 1; + + if !&message[i..].starts_with(b"i=") { + return Err(Error::ProtocolSyncError); + } + + i += 2; + + let iterations = match String::from_utf8_lossy(&message[i..]).parse::() { + Ok(it) => it, + Err(_) => return Err(Error::ProtocolSyncError), + }; + + Ok(Message { + nonce, + salt, + iterations, + }) + } +} + +struct FinalMessage { + value: String, +} + +impl FinalMessage { + pub fn parse(message: &BytesMut) -> Result { + if !message.starts_with(b"v=") { + return Err(Error::ProtocolSyncError); + } + + Ok(FinalMessage { + value: String::from_utf8_lossy(&message[2..]).to_string(), + }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn parse_server_first_message() { + let message = BytesMut::from( + &"r=fyko+d2lbbFgONRv9qkxdawL3rfcNHYJY1ZVvWVs7j,s=QSXCR+Q6sek8bf92,i=4096".as_bytes()[..], + ); + let message = Message::parse(&message).unwrap(); + assert_eq!(message.nonce, "fyko+d2lbbFgONRv9qkxdawL3rfcNHYJY1ZVvWVs7j"); + assert_eq!(message.salt, "QSXCR+Q6sek8bf92"); + assert_eq!(message.iterations, 4096); + } + + #[test] + fn parse_server_last_message() { + let f = FinalMessage::parse(&BytesMut::from( + &"v=U+ppxD5XUKtradnv8e2MkeupiA8FU87Sg8CXzXHDAzw".as_bytes()[..], + )) + .unwrap(); + assert_eq!( + f.value, + "U+ppxD5XUKtradnv8e2MkeupiA8FU87Sg8CXzXHDAzw".to_string() + ); + } + + // recorded auth exchange from psql + #[test] + fn exchange() { + let password = "foobar"; + let nonce = "9IZ2O01zb9IgiIZ1WJ/zgpJB"; + + let client_first = "n,,n=,r=9IZ2O01zb9IgiIZ1WJ/zgpJB"; + let server_first = + "r=9IZ2O01zb9IgiIZ1WJ/zgpJBjx/oIRLs02gGSHcw1KEty3eY,s=fs3IXBy7U7+IvVjZ,i\ + =4096"; + let client_final = + "c=biws,r=9IZ2O01zb9IgiIZ1WJ/zgpJBjx/oIRLs02gGSHcw1KEty3eY,p=AmNKosjJzS3\ + 1NTlQYNs5BTeQjdHdk7lOflDo5re2an8="; + let server_final = "v=U+ppxD5XUKtradnv8e2MkeupiA8FU87Sg8CXzXHDAzw="; + + let mut scram = ScramSha256::from_nonce(password, nonce); + + let message = scram.message(); + assert_eq!(std::str::from_utf8(&message).unwrap(), client_first); + + let result = scram + .update(&BytesMut::from(&server_first.as_bytes()[..])) + .unwrap(); + assert_eq!(std::str::from_utf8(&result).unwrap(), client_final); + + scram + .finish(&BytesMut::from(&server_final.as_bytes()[..])) + .unwrap(); + } +} diff --git a/src/server.rs b/src/server.rs index 0dd051e59..3670af9b0 100644 --- a/src/server.rs +++ b/src/server.rs @@ -12,6 +12,7 @@ use crate::config::{Address, User}; use crate::constants::*; use crate::errors::Error; use crate::messages::*; +use crate::scram::ScramSha256; use crate::stats::Reporter; use crate::ClientServerMap; @@ -89,6 +90,8 @@ impl Server { // We'll be handling multiple packets, but they will all be structured the same. // We'll loop here until this exchange is complete. + let mut scram = ScramSha256::new(&user.password); + loop { let code = match stream.read_u8().await { Ok(code) => code as char, @@ -130,6 +133,83 @@ impl Server { AUTHENTICATION_SUCCESSFUL => (), + SASL => { + debug!("Starting SASL authentication"); + let sasl_len = (len - 8) as usize; + let mut sasl_auth = vec![0u8; sasl_len]; + match stream.read_exact(&mut sasl_auth).await { + Ok(_) => (), + Err(_) => return Err(Error::SocketError), + }; + + let sasl_type = String::from_utf8_lossy(&sasl_auth[..sasl_len - 2]); + + if sasl_type == SCRAM_SHA_256 { + debug!("Using {}", SCRAM_SHA_256); + + // Send client message + let sasl_response = scram.message(); + let mut res = BytesMut::new(); + res.put_u8(b'p'); + res.put_i32( + 4 + SCRAM_SHA_256.len() as i32 + + 1 + + sasl_response.len() as i32 + + 4, + ); + res.put_slice(&format!("{}\0", SCRAM_SHA_256).as_bytes()[..]); + res.put_i32(sasl_response.len() as i32); + res.put(sasl_response); + + write_all(&mut stream, res).await?; + } else { + error!("Unsupported SCRAM version: {}", sasl_type); + return Err(Error::ServerError); + } + } + + SASL_CONTINUE => { + trace!("Continuing SASL"); + + let mut sasl_data = vec![0u8; (len - 8) as usize]; + + match stream.read_exact(&mut sasl_data).await { + Ok(_) => (), + Err(_) => return Err(Error::SocketError), + }; + + let msg = BytesMut::from(&sasl_data[..]); + let sasl_response = scram.update(&msg)?; + + let mut res = BytesMut::new(); + res.put_u8(b'p'); + res.put_i32(4 + sasl_response.len() as i32); + res.put(sasl_response); + + write_all(&mut stream, res).await?; + } + + SASL_FINAL => { + trace!("Final SASL"); + + let mut sasl_final = vec![0u8; len as usize - 8]; + match stream.read_exact(&mut sasl_final).await { + Ok(_) => (), + Err(_) => return Err(Error::SocketError), + }; + + match scram.finish(&BytesMut::from(&sasl_final[..])) { + Ok(_) => { + debug!("SASL authentication successful"); + } + + Err(err) => { + debug!("SASL authentication failed"); + return Err(err); + } + }; + } + _ => { error!("Unsupported authentication mechanism: {}", auth_code); return Err(Error::ServerError); From b3455a5630d80fc67866f2261b284fbaf0c9c85a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 20 Jun 2022 06:15:54 -0700 Subject: [PATCH 140/551] Client md5 auth and clean up scram (#77) * client md5 auth and clean up scram * add pw * add user * add user * log --- .circleci/run_tests.sh | 41 +++++++------- Cargo.lock | 2 +- Cargo.toml | 2 +- src/client.rs | 45 +++++++++++++-- src/errors.rs | 1 + src/messages.rs | 76 ++++++++++++++++++++++--- src/scram.rs | 123 ++++++++++++++++++++++------------------- src/server.rs | 18 ++++-- 8 files changed, 213 insertions(+), 95 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index f325d1665..66fcb7922 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -13,6 +13,7 @@ function start_pgcat() { # Setup the database with shards and user PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql + PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard0 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard2 -i @@ -30,26 +31,28 @@ toxiproxy-cli create -l 127.0.0.1:5433 -u 127.0.0.1:5432 postgres_replica start_pgcat "info" +export PGPASSWORD=sharding_user + # pgbench test -pgbench -i -h 127.0.0.1 -p 6432 -pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol simple -f tests/pgbench/simple.sql -pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended +pgbench -U sharding_user -i -h 127.0.0.1 -p 6432 +pgbench -U sharding_user -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol simple -f tests/pgbench/simple.sql +pgbench -U sharding_user -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended # COPY TO STDOUT test -psql -h 127.0.0.1 -p 6432 -c 'COPY (SELECT * FROM pgbench_accounts LIMIT 15) TO STDOUT;' > /dev/null +psql -U sharding_user -h 127.0.0.1 -p 6432 -c 'COPY (SELECT * FROM pgbench_accounts LIMIT 15) TO STDOUT;' > /dev/null # Query cancellation test -(psql -h 127.0.0.1 -p 6432 -c 'SELECT pg_sleep(5)' || true) & +(psql -U sharding_user -h 127.0.0.1 -p 6432 -c 'SELECT pg_sleep(5)' || true) & killall psql -s SIGINT # Sharding insert -psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_insert.sql +psql -U sharding_user -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_insert.sql # Sharding select -psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql > /dev/null +psql -U sharding_user -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_select.sql > /dev/null # Replica/primary selection & more sharding tests -psql -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null +psql -U sharding_user -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null # # ActiveRecord tests @@ -61,15 +64,15 @@ cd tests/ruby && \ cd ../.. # Admin tests -psql -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/null -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW CONFIG' > /dev/null -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' > /dev/null -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW LISTS' > /dev/null -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW POOLS' > /dev/null -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW VERSION' > /dev/null -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c "SET client_encoding TO 'utf8'" > /dev/null # will ignore -(! psql -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) +psql -U sharding_user -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/null +psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null +psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW CONFIG' > /dev/null +psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' > /dev/null +psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW LISTS' > /dev/null +psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW POOLS' > /dev/null +psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW VERSION' > /dev/null +psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c "SET client_encoding TO 'utf8'" > /dev/null # will ignore +(! psql -U sharding_user -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) # Start PgCat in debug to demonstrate failover better start_pgcat "trace" @@ -79,7 +82,7 @@ toxiproxy-cli toxic add -t latency -a latency=300 postgres_replica sleep 1 # Note the failover in the logs -timeout 5 psql -e -h 127.0.0.1 -p 6432 <<-EOF +timeout 5 psql -U sharding_user -e -h 127.0.0.1 -p 6432 <<-EOF SELECT 1; SELECT 1; SELECT 1; @@ -97,7 +100,7 @@ sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' pgcat.toml kill -SIGHUP $(pgrep pgcat) # Prepared statements that will only work in session mode -pgbench -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol prepared +pgbench -U sharding_user -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol prepared # Attempt clean shut down killall pgcat -s SIGINT diff --git a/Cargo.lock b/Cargo.lock index 51bbc6e58..668f421da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -368,7 +368,7 @@ dependencies = [ [[package]] name = "pgcat" -version = "0.1.0-beta2" +version = "0.2.0-beta1" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index ae18d7b73..924b9cb02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "0.2.0-beta1" +version = "0.2.1-beta1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/client.rs b/src/client.rs index 2c96ccc2d..08b704920 100644 --- a/src/client.rs +++ b/src/client.rs @@ -72,9 +72,9 @@ impl Client { server_info: BytesMut, stats: Reporter, ) -> Result { - let config = get_config(); + let config = get_config().clone(); let transaction_mode = config.general.pool_mode.starts_with("t"); - drop(config); + // drop(config); loop { trace!("Waiting for StartupMessage"); @@ -108,14 +108,51 @@ impl Client { // Regular startup message. PROTOCOL_VERSION_NUMBER => { trace!("Got StartupMessage"); - - // TODO: perform actual auth. let parameters = parse_startup(bytes.clone())?; // Generate random backend ID and secret key let process_id: i32 = rand::random(); let secret_key: i32 = rand::random(); + // Perform MD5 authentication. + // TODO: Add SASL support. + let salt = md5_challenge(&mut stream).await?; + + let code = match stream.read_u8().await { + Ok(p) => p, + Err(_) => return Err(Error::SocketError), + }; + + // PasswordMessage + if code as char != 'p' { + debug!("Expected p, got {}", code as char); + return Err(Error::ProtocolSyncError); + } + + let len = match stream.read_i32().await { + Ok(len) => len, + Err(_) => return Err(Error::SocketError), + }; + + let mut password_response = vec![0u8; (len - 4) as usize]; + + match stream.read_exact(&mut password_response).await { + Ok(_) => (), + Err(_) => return Err(Error::SocketError), + }; + + // Compare server and client hashes. + let password_hash = + md5_hash_password(&config.user.name, &config.user.password, &salt); + + if password_hash != password_response { + debug!("Password authentication failed"); + wrong_password(&mut stream, &config.user.name).await?; + return Err(Error::ClientError); + } + + debug!("Password authentication successful"); + auth_ok(&mut stream).await?; write_all(&mut stream, server_info).await?; backend_key_data(&mut stream, process_id, secret_key).await?; diff --git a/src/errors.rs b/src/errors.rs index b42d32197..b07d50883 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -9,4 +9,5 @@ pub enum Error { ServerError, BadConfig, AllServersDown, + ClientError, } diff --git a/src/messages.rs b/src/messages.rs index 3420bb4ff..993545bbb 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -40,6 +40,26 @@ pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { Ok(write_all(stream, auth_ok).await?) } +/// Generate md5 password challenge. +pub async fn md5_challenge(stream: &mut TcpStream) -> Result<[u8; 4], Error> { + // let mut rng = rand::thread_rng(); + let salt: [u8; 4] = [ + rand::random(), + rand::random(), + rand::random(), + rand::random(), + ]; + + let mut res = BytesMut::new(); + res.put_u8(b'R'); + res.put_i32(12); + res.put_i32(5); // MD5 + res.put_slice(&salt[..]); + + write_all(stream, res).await?; + Ok(salt) +} + /// Give the client the process_id and secret we generated /// used in query cancellation. pub async fn backend_key_data( @@ -160,14 +180,8 @@ pub fn parse_startup(bytes: BytesMut) -> Result, Error> Ok(result) } -/// Send password challenge response to the server. -/// This is the MD5 challenge. -pub async fn md5_password( - stream: &mut TcpStream, - user: &str, - password: &str, - salt: &[u8], -) -> Result<(), Error> { +/// Create md5 password hash given a salt. +pub fn md5_hash_password(user: &str, password: &str, salt: &[u8]) -> Vec { let mut md5 = Md5::new(); // First pass @@ -186,6 +200,19 @@ pub async fn md5_password( .collect::>(); password.push(0); + password +} + +/// Send password challenge response to the server. +/// This is the MD5 challenge. +pub async fn md5_password( + stream: &mut TcpStream, + user: &str, + password: &str, + salt: &[u8], +) -> Result<(), Error> { + let password = md5_hash_password(user, password, salt); + let mut message = BytesMut::with_capacity(password.len() as usize + 5); message.put_u8(b'p'); @@ -264,6 +291,39 @@ pub async fn error_response(stream: &mut OwnedWriteHalf, message: &str) -> Resul Ok(write_all_half(stream, res).await?) } +pub async fn wrong_password(stream: &mut TcpStream, user: &str) -> Result<(), Error> { + let mut error = BytesMut::new(); + + // Error level + error.put_u8(b'S'); + error.put_slice(&b"FATAL\0"[..]); + + // Error level (non-translatable) + error.put_u8(b'V'); + error.put_slice(&b"FATAL\0"[..]); + + // Error code: not sure how much this matters. + error.put_u8(b'C'); + error.put_slice(&b"28P01\0"[..]); // system_error, see Appendix A. + + // The short error message. + error.put_u8(b'M'); + error.put_slice(&format!("password authentication failed for user \"{}\"\0", user).as_bytes()); + + // No more fields follow. + error.put_u8(0); + + // Compose the two message reply. + let mut res = BytesMut::new(); + + res.put_u8(b'E'); + res.put_i32(error.len() as i32 + 4); + + res.put(error); + + write_all(stream, res).await +} + /// Respond to a SHOW SHARD command. pub async fn show_response( stream: &mut OwnedWriteHalf, diff --git a/src/scram.rs b/src/scram.rs index 58096fa8e..514ed7a12 100644 --- a/src/scram.rs +++ b/src/scram.rs @@ -1,5 +1,6 @@ -// SCRAM authentication...largely copy/pasted from -// https://github.com/sfackler/rust-postgres/. +// SCRAM-SHA-256 authentication. Heavily inspired by +// https://github.com/sfackler/rust-postgres/ +// SASL implementation. use bytes::BytesMut; use hmac::{Hmac, Mac}; @@ -12,6 +13,8 @@ use std::fmt::Write; use crate::constants::*; use crate::errors::Error; +/// Normalize a password string. Postgres +/// passwords don't have to be UTF-8. fn normalize(pass: &[u8]) -> Vec { let pass = match std::str::from_utf8(pass) { Ok(pass) => pass, @@ -24,6 +27,8 @@ fn normalize(pass: &[u8]) -> Vec { } } +/// Keep the SASL state through the exchange. +/// It takes 3 messages to complete the authentication. pub struct ScramSha256 { password: String, salted_password: [u8; 32], @@ -33,6 +38,8 @@ pub struct ScramSha256 { } impl ScramSha256 { + /// Create the Scram state from a password. It'll automatically + /// generate a nonce. pub fn new(password: &str) -> ScramSha256 { let mut rng = rand::thread_rng(); let nonce = (0..NONCE_LENGTH) @@ -48,6 +55,7 @@ impl ScramSha256 { Self::from_nonce(password, &nonce) } + /// Used for testing. pub fn from_nonce(password: &str, nonce: &str) -> ScramSha256 { let message = BytesMut::from(&format!("{}n=,r={}", "n,,", nonce).as_bytes()[..]); @@ -60,15 +68,16 @@ impl ScramSha256 { } } + /// Get the current state of the SASL authentication. pub fn message(&mut self) -> BytesMut { self.message.clone() } + /// Update the state with message received from server. pub fn update(&mut self, message: &BytesMut) -> Result { let server_message = Message::parse(message)?; if !server_message.nonce.starts_with(&self.nonce) { - // trace!("Bad server nonce"); return Err(Error::ProtocolSyncError); } @@ -82,28 +91,39 @@ impl ScramSha256 { &salt, server_message.iterations, ); + + // Save for verification of final server message. self.salted_password = salted_password; - let mut hmac = Hmac::::new_from_slice(&salted_password) - .expect("HMAC is able to accept all key sizes"); + let mut hmac = match Hmac::::new_from_slice(&salted_password) { + Ok(hmac) => hmac, + Err(_) => return Err(Error::ServerError), + }; + hmac.update(b"Client Key"); + let client_key = hmac.finalize().into_bytes(); let mut hash = Sha256::default(); hash.update(client_key.as_slice()); - let stored_key = hash.finalize_fixed(); + let stored_key = hash.finalize_fixed(); let mut cbind_input = vec![]; cbind_input.extend("n,,".as_bytes()); + let cbind_input = base64::encode(&cbind_input); self.message.clear(); - write!( + + // Start writing the client reply. + match write!( &mut self.message, "c={},r={}", cbind_input, server_message.nonce - ) - .unwrap(); + ) { + Ok(_) => (), + Err(_) => return Err(Error::ServerError), + }; let auth_message = format!( "n=,r={},{},{}", @@ -112,23 +132,32 @@ impl ScramSha256 { String::from_utf8_lossy(&self.message[..]) ); - let mut hmac = Hmac::::new_from_slice(&stored_key) - .expect("HMAC is able to accept all key sizes"); + let mut hmac = match Hmac::::new_from_slice(&stored_key) { + Ok(hmac) => hmac, + Err(_) => return Err(Error::ServerError), + }; hmac.update(auth_message.as_bytes()); + + // Save the auth message for server final message verification. + self.auth_message = auth_message; + let client_signature = hmac.finalize().into_bytes(); + // Sign the client proof. let mut client_proof = client_key; for (proof, signature) in client_proof.iter_mut().zip(client_signature) { *proof ^= signature; } - write!(&mut self.message, ",p={}", base64::encode(&*client_proof)).unwrap(); - - self.auth_message = auth_message; + match write!(&mut self.message, ",p={}", base64::encode(&*client_proof)) { + Ok(_) => (), + Err(_) => return Err(Error::ServerError), + }; Ok(self.message.clone()) } + /// Verify final server message. pub fn finish(&mut self, message: &BytesMut) -> Result<(), Error> { let final_message = FinalMessage::parse(message)?; @@ -137,13 +166,17 @@ impl ScramSha256 { Err(_) => return Err(Error::ProtocolSyncError), }; - let mut hmac = Hmac::::new_from_slice(&self.salted_password) - .expect("HMAC is able to accept all key sizes"); + let mut hmac = match Hmac::::new_from_slice(&self.salted_password) { + Ok(hmac) => hmac, + Err(_) => return Err(Error::ServerError), + }; hmac.update(b"Server Key"); let server_key = hmac.finalize().into_bytes(); - let mut hmac = Hmac::::new_from_slice(&server_key) - .expect("HMAC is able to accept all key sizes"); + let mut hmac = match Hmac::::new_from_slice(&server_key) { + Ok(hmac) => hmac, + Err(_) => return Err(Error::ServerError), + }; hmac.update(self.auth_message.as_bytes()); match hmac.verify_slice(&verifier) { @@ -152,7 +185,7 @@ impl ScramSha256 { } } - // https://github.com/sfackler/rust-postgres/blob/c3a029e60c1c0bd0be947049859b8fa5bd5ac220/postgres-protocol/src/authentication/sasl.rs#L35 + /// Hash the password with the salt i-times. fn hi(str: &[u8], salt: &[u8], i: u32) -> [u8; 32] { let mut hmac = Hmac::::new_from_slice(str).expect("HMAC is able to accept all key sizes"); @@ -176,7 +209,7 @@ impl ScramSha256 { } } -#[derive(Default, Debug)] +/// Parse the server challenge. struct Message { nonce: String, salt: String, @@ -184,47 +217,21 @@ struct Message { } impl Message { + /// Parse the server SASL challenge. fn parse(message: &BytesMut) -> Result { - if !message.starts_with(b"r=") { - return Err(Error::ProtocolSyncError); - } - - let mut i = 2; - - while message[i] != b',' && i < message.len() { - i += 1; - } - - let nonce = String::from_utf8_lossy(&message[2..i]).to_string(); + let parts = String::from_utf8_lossy(&message[..]) + .split(",") + .map(|s| s.to_string()) + .collect::>(); - // Skip the , - i += 1; - - if !&message[i..].starts_with(b"s=") { - return Err(Error::ProtocolSyncError); - } - - // Skip the s= - i += 2; - - let s = i; - while message[i] != b',' && i < message.len() { - i += 1; - } - - let salt = String::from_utf8_lossy(&message[s..i]).to_string(); - - // Skip the , - i += 1; - - if !&message[i..].starts_with(b"i=") { + if parts.len() != 3 { return Err(Error::ProtocolSyncError); } - i += 2; - - let iterations = match String::from_utf8_lossy(&message[i..]).parse::() { - Ok(it) => it, + let nonce = str::replace(&parts[0], "r=", ""); + let salt = str::replace(&parts[1], "s=", ""); + let iterations = match str::replace(&parts[2], "i=", "").parse::() { + Ok(iterations) => iterations, Err(_) => return Err(Error::ProtocolSyncError), }; @@ -236,13 +243,15 @@ impl Message { } } +/// Parse server final validation message. struct FinalMessage { value: String, } impl FinalMessage { + /// Parse the server final validation message. pub fn parse(message: &BytesMut) -> Result { - if !message.starts_with(b"v=") { + if !message.starts_with(b"v=") || message.len() < 4 { return Err(Error::ProtocolSyncError); } diff --git a/src/server.rs b/src/server.rs index 3670af9b0..b20d15335 100644 --- a/src/server.rs +++ b/src/server.rs @@ -137,6 +137,7 @@ impl Server { debug!("Starting SASL authentication"); let sasl_len = (len - 8) as usize; let mut sasl_auth = vec![0u8; sasl_len]; + match stream.read_exact(&mut sasl_auth).await { Ok(_) => (), Err(_) => return Err(Error::SocketError), @@ -147,16 +148,22 @@ impl Server { if sasl_type == SCRAM_SHA_256 { debug!("Using {}", SCRAM_SHA_256); - // Send client message + // Generate client message. let sasl_response = scram.message(); + + // SASLInitialResponse (F) let mut res = BytesMut::new(); res.put_u8(b'p'); + + // length + String length + length + length of sasl response res.put_i32( - 4 + SCRAM_SHA_256.len() as i32 - + 1 - + sasl_response.len() as i32 - + 4, + 4 // i32 size + + SCRAM_SHA_256.len() as i32 // length of SASL version string, + + 1 // Null terminator for the SASL version string, + + 4 // i32 size + + sasl_response.len() as i32, // length of SASL response ); + res.put_slice(&format!("{}\0", SCRAM_SHA_256).as_bytes()[..]); res.put_i32(sasl_response.len() as i32); res.put(sasl_response); @@ -181,6 +188,7 @@ impl Server { let msg = BytesMut::from(&sasl_data[..]); let sasl_response = scram.update(&msg)?; + // SASLResponse let mut res = BytesMut::new(); res.put_u8(b'p'); res.put_i32(4 + sasl_response.len() as i32); From 71e252e7a569990b2ca7dc1d8c4c52b344362926 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 20 Jun 2022 06:20:12 -0700 Subject: [PATCH 141/551] readme --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 12b84c747..5b54a7fc3 100644 --- a/README.md +++ b/README.md @@ -9,19 +9,19 @@ PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover su **Beta**: looking for beta testers, see [#35](https://github.com/levkk/pgcat/issues/35). ## Features -| **Feature** | **Status** | **Comments** | -|--------------------------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| -| Transaction pooling | :white_check_mark: | Identical to PgBouncer. | -| Session pooling | :white_check_mark: | Identical to PgBouncer. | -| `COPY` support | :white_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | -| Query cancellation | :white_check_mark: | Supported both in transaction and session pooling modes. | -| Load balancing of read queries | :white_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | -| Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | -| Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | -| Statistics | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. | -| Live configuration reloading | :white_check_mark: | Reload supported settings with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)` or `RELOAD` query issued to the admin database. | -| Client authentication | :x: :wrench: | On the roadmap; currently all clients are allowed to connect and one user is used to connect to Postgres. | -| Admin database | :white_check_mark: | The admin database, similar to PgBouncer's, allows to query for statistics and reload the configuration. | +| **Feature** | **Status** | **Comments** | +|--------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| +| Transaction pooling | :white_check_mark: | Identical to PgBouncer. | +| Session pooling | :white_check_mark: | Identical to PgBouncer. | +| `COPY` support | :white_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | +| Query cancellation | :white_check_mark: | Supported both in transaction and session pooling modes. | +| Load balancing of read queries | :white_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | +| Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | +| Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | +| Statistics | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. | +| Live configuration reloading | :white_check_mark: | Reload supported settings with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)` or `RELOAD` query issued to the admin database. | +| Client authentication | :white_check_mark: :wrench: | MD5 password authentication is supported, SCRAM is on the roadmap; one user is used to connect to Postgres with both SCRAM and MD5 supported. | +| Admin database | :white_check_mark: | The admin database, similar to PgBouncer's, allows to query for statistics and reload the configuration. | ## Deployment From 337a2c9289572b9aa5c55e1acf1e423e119235d2 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 24 Jun 2022 14:52:38 -0700 Subject: [PATCH 142/551] Live reloading entire config and bug fixes (#84) * Support reloading the entire config (including sharding logic) without restart. * Fix bug incorrectly handing error reporting when the shard is set incorrectly via SET SHARD TO command. selected wrong shard and the connection keep reporting fatal #80. * Fix total_received and avg_recv admin database statistics. * Enabling the query parser by default. * More tests. --- .circleci/pgcat.toml | 2 +- .circleci/run_tests.sh | 12 +- Cargo.lock | 2 +- Cargo.toml | 4 +- pgcat.toml | 2 +- src/admin.rs | 33 ++-- src/client.rs | 106 ++++++++---- src/config.rs | 46 ++++- src/main.rs | 64 +++---- src/pool.rs | 102 ++++++++--- src/query_router.rs | 160 ++++++++++++------ src/stats.rs | 28 ++- tests/pgbench/simple.sql | 11 ++ .../query_routing_test_primary_replica.sql | 9 + 14 files changed, 393 insertions(+), 188 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index d1dd19db5..ba3b87515 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -87,7 +87,7 @@ default_role = "any" # every incoming query to determine if it's a read or a write. # If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, # we'll direct it to the primary. -query_parser_enabled = false +query_parser_enabled = true # If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for # load balancing of read queries. Otherwise, the primary will only be used for write diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 66fcb7922..a0e23f0ac 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -42,7 +42,15 @@ pgbench -U sharding_user -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended psql -U sharding_user -h 127.0.0.1 -p 6432 -c 'COPY (SELECT * FROM pgbench_accounts LIMIT 15) TO STDOUT;' > /dev/null # Query cancellation test -(psql -U sharding_user -h 127.0.0.1 -p 6432 -c 'SELECT pg_sleep(5)' || true) & +(psql -U sharding_user -h 127.0.0.1 -p 6432 -c 'SELECT pg_sleep(50)' || true) & +sleep 1 +killall psql -s SIGINT + +# Reload pool (closing unused server connections) +psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' + +(psql -U sharding_user -h 127.0.0.1 -p 6432 -c 'SELECT pg_sleep(50)' || true) & +sleep 1 killall psql -s SIGINT # Sharding insert @@ -94,7 +102,7 @@ toxiproxy-cli toxic remove --toxicName latency_downstream postgres_replica start_pgcat "info" # Test session mode (and config reload) -sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' pgcat.toml +sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' .circleci/pgcat.toml # Reload config test kill -SIGHUP $(pgrep pgcat) diff --git a/Cargo.lock b/Cargo.lock index 668f421da..87d11be7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -368,7 +368,7 @@ dependencies = [ [[package]] name = "pgcat" -version = "0.2.0-beta1" +version = "0.4.0-beta1" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 924b9cb02..fa63c0e8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "0.2.1-beta1" +version = "0.4.0-beta1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -28,4 +28,4 @@ parking_lot = "0.11" hmac = "0.12" sha2 = "0.10" base64 = "0.13" -stringprep = "0.1" \ No newline at end of file +stringprep = "0.1" diff --git a/pgcat.toml b/pgcat.toml index 5311bd74d..435dda95c 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -87,7 +87,7 @@ default_role = "any" # every incoming query to determine if it's a read or a write. # If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, # we'll direct it to the primary. -query_parser_enabled = false +query_parser_enabled = true # If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for # load balancing of read queries. Otherwise, the primary will only be used for write diff --git a/src/admin.rs b/src/admin.rs index c4675703d..b7a5b6fd8 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -4,17 +4,19 @@ use log::{info, trace}; use std::collections::HashMap; use tokio::net::tcp::OwnedWriteHalf; -use crate::config::{get_config, parse}; +use crate::config::{get_config, reload_config}; use crate::errors::Error; use crate::messages::*; use crate::pool::ConnectionPool; use crate::stats::get_stats; +use crate::ClientServerMap; /// Handle admin client. pub async fn handle_admin( stream: &mut OwnedWriteHalf, mut query: BytesMut, pool: ConnectionPool, + client_server_map: ClientServerMap, ) -> Result<(), Error> { let code = query.get_u8() as char; @@ -34,7 +36,7 @@ pub async fn handle_admin( show_stats(stream, &pool).await } else if query.starts_with("RELOAD") { trace!("RELOAD"); - reload(stream).await + reload(stream, client_server_map).await } else if query.starts_with("SHOW CONFIG") { trace!("SHOW CONFIG"); show_config(stream).await @@ -143,10 +145,7 @@ async fn show_version(stream: &mut OwnedWriteHalf) -> Result<(), Error> { /// Show utilization of connection pools for each shard and replicas. async fn show_pools(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { let stats = get_stats(); - let config = { - let guard = get_config(); - &*guard.clone() - }; + let config = get_config(); let columns = vec![ ("database", DataType::Text), @@ -199,9 +198,7 @@ async fn show_pools(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul /// Show shards and replicas. async fn show_databases(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { - let guard = get_config(); - let config = &*guard.clone(); - drop(guard); + let config = get_config(); // Columns let columns = vec![ @@ -266,17 +263,15 @@ async fn ignore_set(stream: &mut OwnedWriteHalf) -> Result<(), Error> { } /// Reload the configuration file without restarting the process. -async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { +async fn reload( + stream: &mut OwnedWriteHalf, + client_server_map: ClientServerMap, +) -> Result<(), Error> { info!("Reloading config"); - let config = get_config(); - let path = config.path.clone().unwrap(); - - parse(&path).await?; - - let config = get_config(); + reload_config(client_server_map).await?; - config.show(); + get_config().show(); let mut res = BytesMut::new(); @@ -292,10 +287,8 @@ async fn reload(stream: &mut OwnedWriteHalf) -> Result<(), Error> { /// Shows current configuration. async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { - let guard = get_config(); - let config = &*guard.clone(); + let config = &get_config(); let config: HashMap = config.into(); - drop(guard); // Configs that cannot be changed without restarting. let immutables = ["host", "port", "connect_timeout"]; diff --git a/src/client.rs b/src/client.rs index 08b704920..b53bd3352 100644 --- a/src/client.rs +++ b/src/client.rs @@ -13,10 +13,10 @@ use crate::config::get_config; use crate::constants::*; use crate::errors::Error; use crate::messages::*; -use crate::pool::{ClientServerMap, ConnectionPool}; +use crate::pool::{get_pool, ClientServerMap}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; -use crate::stats::Reporter; +use crate::stats::{get_reporter, Reporter}; /// The client state. One of these is created per client. pub struct Client { @@ -69,12 +69,11 @@ impl Client { pub async fn startup( mut stream: TcpStream, client_server_map: ClientServerMap, - server_info: BytesMut, - stats: Reporter, ) -> Result { - let config = get_config().clone(); - let transaction_mode = config.general.pool_mode.starts_with("t"); - // drop(config); + let config = get_config(); + let transaction_mode = config.general.pool_mode == "transaction"; + let stats = get_reporter(); + loop { trace!("Waiting for StartupMessage"); @@ -154,9 +153,10 @@ impl Client { debug!("Password authentication successful"); auth_ok(&mut stream).await?; - write_all(&mut stream, server_info).await?; + write_all(&mut stream, get_pool().server_info()).await?; backend_key_data(&mut stream, process_id, secret_key).await?; ready_for_query(&mut stream).await?; + trace!("Startup OK"); let database = parameters @@ -221,7 +221,7 @@ impl Client { } /// Handle a connected and authenticated client. - pub async fn handle(&mut self, mut pool: ConnectionPool) -> Result<(), Error> { + pub async fn handle(&mut self) -> Result<(), Error> { // The client wants to cancel a query it has issued previously. if self.cancel_mode { trace!("Sending CancelRequest"); @@ -252,13 +252,19 @@ impl Client { return Ok(Server::cancel(&address, &port, process_id, secret_key).await?); } + // The query router determines where the query is going to go, + // e.g. primary, replica, which shard. let mut query_router = QueryRouter::new(); + let mut round_robin = 0; // Our custom protocol loop. // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocol. loop { - trace!("Client idle, waiting for message"); + trace!( + "Client idle, waiting for message, transaction mode: {}", + self.transaction_mode + ); // Read a complete message from the client, which normally would be // either a `Q` (query) or `P` (prepare, extended protocol). @@ -267,32 +273,63 @@ impl Client { // SET SHARDING KEY TO 'bigint'; let mut message = read_message(&mut self.read).await?; + // Get a pool instance referenced by the most up-to-date + // pointer. This ensures we always read the latest config + // when starting a query. + let mut pool = get_pool(); + // Avoid taking a server if the client just wants to disconnect. if message[0] as char == 'X' { - trace!("Client disconnecting"); + debug!("Client disconnecting"); return Ok(()); } // Handle admin database queries. if self.admin { - trace!("Handling admin command"); - handle_admin(&mut self.write, message, pool.clone()).await?; + debug!("Handling admin command"); + handle_admin( + &mut self.write, + message, + pool.clone(), + self.client_server_map.clone(), + ) + .await?; continue; } + let current_shard = query_router.shard(); + // Handle all custom protocol commands, if any. match query_router.try_execute_command(message.clone()) { // Normal query, not a custom command. - None => { - // Attempt to infer which server we want to query, i.e. primary or replica. - if query_router.query_parser_enabled() && query_router.role() == None { - query_router.infer_role(message.clone()); - } - } + None => (), // SET SHARD TO Some((Command::SetShard, _)) => { - custom_protocol_response_ok(&mut self.write, "SET SHARD").await?; + // Selected shard is not configured. + if query_router.shard() >= pool.shards() { + // Set the shard back to what it was. + query_router.set_shard(current_shard); + + error_response( + &mut self.write, + &format!( + "shard {} is more than configured {}, staying on shard {}", + query_router.shard(), + pool.shards(), + current_shard, + ), + ) + .await?; + } else { + custom_protocol_response_ok(&mut self.write, "SET SHARD").await?; + } + continue; + } + + // SET PRIMARY READS TO + Some((Command::SetPrimaryReads, _)) => { + custom_protocol_response_ok(&mut self.write, "SET PRIMARY READS").await?; continue; } @@ -319,27 +356,24 @@ impl Client { show_response(&mut self.write, "shard", &value).await?; continue; } - }; - // Make sure we selected a valid shard. - if query_router.shard() >= pool.shards() { - error_response( - &mut self.write, - &format!( - "shard {} is more than configured {}", - query_router.shard(), - pool.shards() - ), - ) - .await?; - continue; - } + // SHOW PRIMARY READS + Some((Command::ShowPrimaryReads, value)) => { + show_response(&mut self.write, "primary reads", &value).await?; + continue; + } + }; debug!("Waiting for connection from pool"); // Grab a server from the pool. let connection = match pool - .get(query_router.shard(), query_router.role(), self.process_id) + .get( + query_router.shard(), + query_router.role(), + self.process_id, + round_robin, + ) .await { Ok(conn) => { @@ -358,6 +392,8 @@ impl Client { let address = connection.1; let server = &mut *reference; + round_robin += 1; + // Server is assigned to the client in case the client wants to // cancel a query later. server.claim(self.process_id, self.secret_key); diff --git a/src/config.rs b/src/config.rs index f11aa3155..96d5a778f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,5 @@ /// Parse the configuration file. -use arc_swap::{ArcSwap, Guard}; +use arc_swap::ArcSwap; use log::{error, info}; use once_cell::sync::Lazy; use serde_derive::Deserialize; @@ -10,6 +10,7 @@ use tokio::io::AsyncReadExt; use toml; use crate::errors::Error; +use crate::{ClientServerMap, ConnectionPool}; /// Globally available configuration. static CONFIG: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Config::default())); @@ -126,7 +127,7 @@ impl Default for General { } /// Shard configuration. -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, PartialEq)] pub struct Shard { pub servers: Vec<(String, u16, String)>, pub database: String, @@ -161,10 +162,16 @@ impl Default for QueryRouter { } } +fn default_path() -> String { + String::from("pgcat.toml") +} + /// Configuration wrapper. #[derive(Deserialize, Debug, Clone)] pub struct Config { - pub path: Option, + #[serde(default = "default_path")] + pub path: String, + pub general: General, pub user: User, pub shards: HashMap, @@ -174,7 +181,7 @@ pub struct Config { impl Default for Config { fn default() -> Config { Config { - path: Some(String::from("pgcat.toml")), + path: String::from("pgcat.toml"), general: General::default(), user: User::default(), shards: HashMap::from([(String::from("1"), Shard::default())]), @@ -237,6 +244,8 @@ impl Config { ); info!("Connection timeout: {}ms", self.general.connect_timeout); info!("Sharding function: {}", self.query_router.sharding_function); + info!("Primary reads: {}", self.query_router.primary_reads_enabled); + info!("Query router: {}", self.query_router.query_parser_enabled); info!("Number of shards: {}", self.shards.len()); } } @@ -244,8 +253,8 @@ impl Config { /// Get a read-only instance of the configuration /// from anywhere in the app. /// ArcSwap makes this cheap and quick. -pub fn get_config() -> Guard> { - CONFIG.load() +pub fn get_config() -> Config { + (*(*CONFIG.load())).clone() } /// Parse the configuration file located at the path. @@ -357,7 +366,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; - config.path = Some(path.to_string()); + config.path = path.to_string(); // Update the configuration globally. CONFIG.store(Arc::new(config.clone())); @@ -365,6 +374,27 @@ pub async fn parse(path: &str) -> Result<(), Error> { Ok(()) } +pub async fn reload_config(client_server_map: ClientServerMap) -> Result<(), Error> { + let old_config = get_config(); + + match parse(&old_config.path).await { + Ok(()) => (), + Err(err) => { + error!("Config reload error: {:?}", err); + return Err(Error::BadConfig); + } + }; + + let new_config = get_config(); + + if old_config.shards != new_config.shards { + info!("Sharding configuration changed, re-creating server pools"); + ConnectionPool::from_config(client_server_map).await + } else { + Ok(()) + } +} + #[cfg(test)] mod test { use super::*; @@ -377,6 +407,6 @@ mod test { assert_eq!(get_config().shards["1"].servers[0].0, "127.0.0.1"); assert_eq!(get_config().shards["0"].servers[0].2, "primary"); assert_eq!(get_config().query_router.default_role, "any"); - assert_eq!(get_config().path, Some("pgcat.toml".to_string())); + assert_eq!(get_config().path, "pgcat.toml".to_string()); } } diff --git a/src/main.rs b/src/main.rs index c22391edd..70094d883 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Lev Kokotov +// Copyright (c) 2022 Lev Kokotov // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the @@ -34,7 +34,7 @@ extern crate sqlparser; extern crate tokio; extern crate toml; -use log::{error, info}; +use log::{debug, error, info}; use parking_lot::Mutex; use tokio::net::TcpListener; use tokio::{ @@ -59,9 +59,9 @@ mod server; mod sharding; mod stats; -use config::get_config; -use pool::{ClientServerMap, ConnectionPool}; -use stats::{Collector, Reporter}; +use config::{get_config, reload_config}; +use pool::{get_pool, ClientServerMap, ConnectionPool}; +use stats::{Collector, Reporter, REPORTER}; #[tokio::main(worker_threads = 4)] async fn main() { @@ -109,37 +109,39 @@ async fn main() { // Statistics reporting. let (tx, rx) = mpsc::channel(100); + REPORTER.store(Arc::new(Reporter::new(tx.clone()))); // Connection pool that allows to query all shards and replicas. - let mut pool = - ConnectionPool::from_config(client_server_map.clone(), Reporter::new(tx.clone())).await; + match ConnectionPool::from_config(client_server_map.clone()).await { + Ok(_) => (), + Err(err) => { + error!("Pool error: {:?}", err); + return; + } + }; + + let pool = get_pool(); // Statistics collector task. let collector_tx = tx.clone(); + + // Save these for reloading + let reload_client_server_map = client_server_map.clone(); + let addresses = pool.databases(); tokio::task::spawn(async move { let mut stats_collector = Collector::new(rx, collector_tx); stats_collector.collect(addresses).await; }); - // Connect to all servers and validate their versions. - let server_info = match pool.validate().await { - Ok(info) => info, - Err(err) => { - error!("Could not validate connection pool: {:?}", err); - return; - } - }; - info!("Waiting for clients"); + drop(pool); + // Client connection loop. tokio::task::spawn(async move { loop { - let pool = pool.clone(); let client_server_map = client_server_map.clone(); - let server_info = server_info.clone(); - let reporter = Reporter::new(tx.clone()); let (socket, addr) = match listener.accept().await { Ok((socket, addr)) => (socket, addr), @@ -152,12 +154,11 @@ async fn main() { // Handle client. tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); - match client::Client::startup(socket, client_server_map, server_info, reporter) - .await - { + match client::Client::startup(socket, client_server_map).await { Ok(mut client) => { info!("Client {:?} connected", addr); - match client.handle(pool).await { + + match client.handle().await { Ok(()) => { let duration = chrono::offset::Utc::now().naive_utc() - start; @@ -176,7 +177,7 @@ async fn main() { } Err(err) => { - error!("Client failed to login: {:?}", err); + debug!("Client failed to login: {:?}", err); } }; }); @@ -190,16 +191,15 @@ async fn main() { loop { stream.recv().await; + info!("Reloading config"); - match config::parse("pgcat.toml").await { - Ok(_) => { - get_config().show(); - } - Err(err) => { - error!("{:?}", err); - return; - } + + match reload_config(reload_client_server_map.clone()).await { + Ok(_) => (), + Err(_) => continue, }; + + get_config().show(); } }); diff --git a/src/pool.rs b/src/pool.rs index 53803e7fe..3f2f36480 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,9 +1,10 @@ -/// Pooling, failover and banlist. +use arc_swap::ArcSwap; use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection}; use bytes::BytesMut; use chrono::naive::NaiveDateTime; use log::{debug, error, info, warn}; +use once_cell::sync::Lazy; use parking_lot::{Mutex, RwLock}; use std::collections::HashMap; use std::sync::Arc; @@ -12,28 +13,47 @@ use std::time::Instant; use crate::config::{get_config, Address, Role, User}; use crate::errors::Error; use crate::server::Server; -use crate::stats::Reporter; +use crate::stats::{get_reporter, Reporter}; pub type BanList = Arc>>>; pub type ClientServerMap = Arc>>; +/// The connection pool, globally available. +/// This is atomic and safe and read-optimized. +/// The pool is recreated dynamically when the config is reloaded. +pub static POOL: Lazy> = + Lazy::new(|| ArcSwap::from_pointee(ConnectionPool::default())); + /// The globally accessible connection pool. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct ConnectionPool { + /// The pools handled internally by bb8. databases: Vec>>, + + /// The addresses (host, port, role) to handle + /// failover and load balancing deterministically. addresses: Vec>, - round_robin: usize, + + /// List of banned addresses (see above) + /// that should not be queried. banlist: BanList, + + /// The statistics aggregator runs in a separate task + /// and receives stats from clients, servers, and the pool. stats: Reporter, + + /// The server information (K messages) have to be passed to the + /// clients on startup. We pre-connect to all shards and replicas + /// on pool creation and save the K messages here. + server_info: BytesMut, } impl ConnectionPool { /// Construct the connection pool from the configuration. - pub async fn from_config( - client_server_map: ClientServerMap, - stats: Reporter, - ) -> ConnectionPool { + pub async fn from_config(client_server_map: ClientServerMap) -> Result<(), Error> { + let reporter = get_reporter(); let config = get_config(); + let mut shards = Vec::new(); let mut addresses = Vec::new(); let mut banlist = Vec::new(); @@ -44,6 +64,8 @@ impl ConnectionPool { .into_keys() .map(|x| x.to_string()) .collect::>(); + + // Sort by shard number to ensure consistency. shard_ids.sort_by_key(|k| k.parse::().unwrap()); for shard_idx in shard_ids { @@ -82,7 +104,7 @@ impl ConnectionPool { config.user.clone(), &shard.database, client_server_map.clone(), - stats.clone(), + reporter.clone(), ); let pool = Pool::builder() @@ -105,15 +127,28 @@ impl ConnectionPool { } assert_eq!(shards.len(), addresses.len()); - let address_len = addresses.len(); - ConnectionPool { + let mut pool = ConnectionPool { databases: shards, addresses: addresses, - round_robin: rand::random::() % address_len, // Start at a random replica banlist: Arc::new(RwLock::new(banlist)), - stats: stats, - } + stats: reporter, + server_info: BytesMut::new(), + }; + + // Connect to the servers to make sure pool configuration is valid + // before setting it globally. + match pool.validate().await { + Ok(_) => (), + Err(err) => { + error!("Could not validate connection pool: {:?}", err); + return Err(err); + } + }; + + POOL.store(Arc::new(pool.clone())); + + Ok(()) } /// Connect to all shards and grab server information. @@ -121,16 +156,18 @@ impl ConnectionPool { /// when they connect. /// This also warms up the pool for clients that connect when /// the pooler starts up. - pub async fn validate(&mut self) -> Result { + async fn validate(&mut self) -> Result<(), Error> { let mut server_infos = Vec::new(); - let stats = self.stats.clone(); + for shard in 0..self.shards() { + let mut round_robin = 0; + for _ in 0..self.servers(shard) { // To keep stats consistent. let fake_process_id = 0; - let connection = match self.get(shard, None, fake_process_id).await { + let connection = match self.get(shard, None, fake_process_id, round_robin).await { Ok(conn) => conn, Err(err) => { error!("Shard {} down or misconfigured: {:?}", shard, err); @@ -138,10 +175,9 @@ impl ConnectionPool { } }; - let mut proxy = connection.0; + let proxy = connection.0; let address = connection.1; - let server = &mut *proxy; - + let server = &*proxy; let server_info = server.server_info(); stats.client_disconnecting(fake_process_id, address.id); @@ -157,6 +193,7 @@ impl ConnectionPool { } server_infos.push(server_info); + round_robin += 1; } } @@ -166,15 +203,18 @@ impl ConnectionPool { return Err(Error::AllServersDown); } - Ok(server_infos[0].clone()) + self.server_info = server_infos[0].clone(); + + Ok(()) } /// Get a connection from the pool. pub async fn get( &mut self, - shard: usize, - role: Option, - process_id: i32, + shard: usize, // shard number + role: Option, // primary or replica + process_id: i32, // client id + mut round_robin: usize, // round robin offset ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { let now = Instant::now(); let addresses = &self.addresses[shard]; @@ -204,9 +244,9 @@ impl ConnectionPool { while allowed_attempts > 0 { // Round-robin replicas. - self.round_robin += 1; + round_robin += 1; - let index = self.round_robin % addresses.len(); + let index = round_robin % addresses.len(); let address = &addresses[index]; // Make sure you're getting a primary or a replica @@ -218,6 +258,7 @@ impl ConnectionPool { allowed_attempts -= 1; + // Don't attempt to connect to banned servers. if self.is_banned(address, shard, role) { continue; } @@ -390,6 +431,10 @@ impl ConnectionPool { pub fn address(&self, shard: usize, server: usize) -> &Address { &self.addresses[shard][server] } + + pub fn server_info(&self) -> BytesMut { + self.server_info.clone() + } } /// Wrapper for the bb8 connection pool. @@ -470,3 +515,8 @@ impl ManageConnection for ServerPool { conn.is_bad() } } + +/// Get the connection pool +pub fn get_pool() -> ConnectionPool { + (*(*POOL.load())).clone() +} diff --git a/src/query_router.rs b/src/query_router.rs index 1f20079d3..995a25c2b 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -12,12 +12,14 @@ use crate::config::{get_config, Role}; use crate::sharding::{Sharder, ShardingFunction}; /// Regexes used to parse custom commands. -const CUSTOM_SQL_REGEXES: [&str; 5] = [ +const CUSTOM_SQL_REGEXES: [&str; 7] = [ r"(?i)^ *SET SHARDING KEY TO '?([0-9]+)'? *;? *$", r"(?i)^ *SET SHARD TO '?([0-9]+|ANY)'? *;? *$", r"(?i)^ *SHOW SHARD *;? *$", r"(?i)^ *SET SERVER ROLE TO '(PRIMARY|REPLICA|ANY|AUTO|DEFAULT)' *;? *$", r"(?i)^ *SHOW SERVER ROLE *;? *$", + r"(?i)^ *SET PRIMARY READS TO '?(on|off|default)'? *;? *$", + r"(?i)^ *SHOW PRIMARY READS *;? *$", ]; /// Custom commands. @@ -28,6 +30,8 @@ pub enum Command { ShowShard, SetServerRole, ShowServerRole, + SetPrimaryReads, + ShowPrimaryReads, } /// Quickly test for match when a query is received. @@ -38,27 +42,17 @@ static CUSTOM_SQL_REGEX_LIST: OnceCell> = OnceCell::new(); /// The query router. pub struct QueryRouter { - /// By default, queries go here, unless we have better information - /// about what the client wants. - default_server_role: Option, - - /// Number of shards in the cluster. - shards: usize, - /// Which shard we should be talking to right now. active_shard: Option, /// Which server should we be talking to. active_role: Option, - /// Include the primary into the replica pool for reads. - primary_reads_enabled: bool, - - /// Should we try to parse queries to route them to replicas or primary automatically. + /// Should we try to parse queries to route them to replicas or primary automatically query_parser_enabled: bool, - /// Which sharding function we're using. - sharding_function: ShardingFunction, + /// Include the primary into the replica pool for reads. + primary_reads_enabled: bool, } impl QueryRouter { @@ -97,28 +91,11 @@ impl QueryRouter { pub fn new() -> QueryRouter { let config = get_config(); - let default_server_role = match config.query_router.default_role.as_ref() { - "any" => None, - "primary" => Some(Role::Primary), - "replica" => Some(Role::Replica), - _ => unreachable!(), - }; - - let sharding_function = match config.query_router.sharding_function.as_ref() { - "pg_bigint_hash" => ShardingFunction::PgBigintHash, - "sha1" => ShardingFunction::Sha1, - _ => unreachable!(), - }; - QueryRouter { - default_server_role: default_server_role, - shards: config.shards.len(), - - active_role: default_server_role, active_shard: None, - primary_reads_enabled: config.query_router.primary_reads_enabled, + active_role: None, query_parser_enabled: config.query_router.query_parser_enabled, - sharding_function, + primary_reads_enabled: config.query_router.primary_reads_enabled, } } @@ -146,21 +123,48 @@ impl QueryRouter { let matches: Vec<_> = regex_set.matches(&query).into_iter().collect(); + // This is not a custom query, try to infer which + // server it'll go to if the query parser is enabled. if matches.len() != 1 { + debug!("Regular query"); + if self.query_parser_enabled && self.role() == None { + debug!("Inferring role"); + self.infer_role(buf.clone()); + } return None; } + let config = get_config(); + + let sharding_function = match config.query_router.sharding_function.as_ref() { + "pg_bigint_hash" => ShardingFunction::PgBigintHash, + "sha1" => ShardingFunction::Sha1, + _ => unreachable!(), + }; + + let default_server_role = match config.query_router.default_role.as_ref() { + "any" => None, + "primary" => Some(Role::Primary), + "replica" => Some(Role::Replica), + _ => unreachable!(), + }; + let command = match matches[0] { 0 => Command::SetShardingKey, 1 => Command::SetShard, 2 => Command::ShowShard, 3 => Command::SetServerRole, 4 => Command::ShowServerRole, + 5 => Command::SetPrimaryReads, + 6 => Command::ShowPrimaryReads, _ => unreachable!(), }; let mut value = match command { - Command::SetShardingKey | Command::SetShard | Command::SetServerRole => { + Command::SetShardingKey + | Command::SetShard + | Command::SetServerRole + | Command::SetPrimaryReads => { // Capture value. I know this re-runs the regex engine, but I haven't // figured out a better way just yet. I think I can write a single Regex // that matches all 5 custom SQL patterns, but maybe that's not very legible? @@ -187,11 +191,16 @@ impl QueryRouter { } } }, + + Command::ShowPrimaryReads => match self.primary_reads_enabled { + true => String::from("on"), + false => String::from("off"), + }, }; match command { Command::SetShardingKey => { - let sharder = Sharder::new(self.shards, self.sharding_function); + let sharder = Sharder::new(config.shards.len(), sharding_function); let shard = sharder.shard(value.parse::().unwrap()); self.active_shard = Some(shard); value = shard.to_string(); @@ -199,7 +208,7 @@ impl QueryRouter { Command::SetShard => { self.active_shard = match value.to_ascii_uppercase().as_ref() { - "ANY" => Some(rand::random::() % self.shards), + "ANY" => Some(rand::random::() % config.shards.len()), _ => Some(value.parse::().unwrap()), }; } @@ -227,8 +236,8 @@ impl QueryRouter { } "default" => { - self.active_role = self.default_server_role; - self.query_parser_enabled = get_config().query_router.query_parser_enabled; + self.active_role = default_server_role; + self.query_parser_enabled = config.query_router.query_parser_enabled; self.active_role } @@ -236,6 +245,19 @@ impl QueryRouter { }; } + Command::SetPrimaryReads => { + if value == "on" { + debug!("Setting primary reads to on"); + self.primary_reads_enabled = true; + } else if value == "off" { + debug!("Setting primary reads to off"); + self.primary_reads_enabled = false; + } else if value == "default" { + debug!("Setting primary reads to default"); + self.primary_reads_enabled = config.query_router.primary_reads_enabled; + } + } + _ => (), } @@ -330,23 +352,15 @@ impl QueryRouter { } } - /// Reset the router back to defaults. - /// This must be called at the end of every transaction in transaction mode. - pub fn _reset(&mut self) { - self.active_role = self.default_server_role; - self.active_shard = None; + pub fn set_shard(&mut self, shard: usize) { + self.active_shard = Some(shard); } /// Should we attempt to parse queries? + #[allow(dead_code)] pub fn query_parser_enabled(&self) -> bool { self.query_parser_enabled } - - /// Allows to toggle primary reads in tests. - #[allow(dead_code)] - pub fn toggle_primary_reads(&mut self, value: bool) { - self.primary_reads_enabled = value; - } } #[cfg(test)] @@ -369,7 +383,8 @@ mod test { let mut qr = QueryRouter::new(); assert!(qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")) != None); assert_eq!(qr.query_parser_enabled(), true); - qr.toggle_primary_reads(false); + + assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); let queries = vec![ simple_query("SELECT * FROM items WHERE id = 5"), @@ -410,7 +425,7 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); let query = simple_query("SELECT * FROM items WHERE id = 5"); - qr.toggle_primary_reads(true); + assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO on")) != None); assert!(qr.infer_role(query)); assert_eq!(qr.role(), None); @@ -421,7 +436,7 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")); - qr.toggle_primary_reads(false); + assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); let prepared_stmt = BytesMut::from( &b"WITH t AS (SELECT * FROM items WHERE name = $1) SELECT * FROM t WHERE id = $2\0"[..], @@ -450,6 +465,10 @@ mod test { "SET SERVER ROLE TO 'any'", "SET SERVER ROLE TO 'auto'", "SHOW SERVER ROLE", + "SET PRIMARY READS TO 'on'", + "SET PRIMARY READS TO 'off'", + "SET PRIMARY READS TO 'default'", + "SHOW PRIMARY READS", // Lower case "set sharding key to '1'", "set shard to '1'", @@ -459,9 +478,13 @@ mod test { "set server role to 'any'", "set server role to 'auto'", "show server role", + "set primary reads to 'on'", + "set primary reads to 'OFF'", + "set primary reads to 'deFaUlt'", // No quotes "SET SHARDING KEY TO 11235", "SET SHARD TO 15", + "SET PRIMARY READS TO off", // Spaces and semicolon " SET SHARDING KEY TO 11235 ; ", " SET SHARD TO 15; ", @@ -469,18 +492,23 @@ mod test { " SET SERVER ROLE TO 'primary'; ", " SET SERVER ROLE TO 'primary' ; ", " SET SERVER ROLE TO 'primary' ;", + " SET PRIMARY READS TO 'off' ;", ]; // Which regexes it'll match to in the list let matches = [ - 0, 1, 2, 3, 3, 3, 3, 4, 0, 1, 2, 3, 3, 3, 3, 4, 0, 1, 0, 1, 0, 3, 3, 3, + 0, 1, 2, 3, 3, 3, 3, 4, 5, 5, 5, 6, 0, 1, 2, 3, 3, 3, 3, 4, 5, 5, 5, 0, 1, 5, 0, 1, 0, + 3, 3, 3, 5, ]; let list = CUSTOM_SQL_REGEX_LIST.get().unwrap(); let set = CUSTOM_SQL_REGEX_SET.get().unwrap(); for (i, test) in tests.iter().enumerate() { - assert!(list[matches[i]].is_match(test)); + if !list[matches[i]].is_match(test) { + println!("{} does not match {}", test, list[matches[i]]); + assert!(false); + } assert_eq!(set.matches(test).into_iter().collect::>().len(), 1); } @@ -549,6 +577,26 @@ mod test { Some((Command::ShowServerRole, String::from(*role))) ); } + + let primary_reads = ["on", "off", "default"]; + let primary_reads_enabled = ["on", "off", "on"]; + + for (idx, primary_reads) in primary_reads.iter().enumerate() { + assert_eq!( + qr.try_execute_command(simple_query(&format!( + "SET PRIMARY READS TO {}", + primary_reads + ))), + Some((Command::SetPrimaryReads, String::from(*primary_reads))) + ); + assert_eq!( + qr.try_execute_command(simple_query("SHOW PRIMARY READS")), + Some(( + Command::ShowPrimaryReads, + String::from(primary_reads_enabled[idx]) + )) + ); + } } #[test] @@ -556,7 +604,7 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); let query = simple_query("SET SERVER ROLE TO 'auto'"); - qr.toggle_primary_reads(false); + assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); assert!(qr.try_execute_command(query) != None); assert!(qr.query_parser_enabled()); @@ -573,6 +621,6 @@ mod test { assert!(qr.query_parser_enabled()); let query = simple_query("SET SERVER ROLE TO 'default'"); assert!(qr.try_execute_command(query) != None); - assert!(!qr.query_parser_enabled()); + assert!(qr.query_parser_enabled()); } } diff --git a/src/stats.rs b/src/stats.rs index e44578d98..7454c583f 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,9 +1,13 @@ +use arc_swap::ArcSwap; /// Statistics and reporting. use log::info; use once_cell::sync::Lazy; use parking_lot::Mutex; use std::collections::HashMap; -use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::sync::mpsc::{channel, Receiver, Sender}; + +pub static REPORTER: Lazy> = + Lazy::new(|| ArcSwap::from_pointee(Reporter::default())); /// Latest stats updated every second; used in SHOW STATS and other admin commands. static LATEST_STATS: Lazy>>> = @@ -60,6 +64,13 @@ pub struct Reporter { tx: Sender, } +impl Default for Reporter { + fn default() -> Reporter { + let (tx, _rx) = channel(5); + Reporter { tx } + } +} + impl Reporter { /// Create a new Reporter instance. pub fn new(tx: Sender) -> Reporter { @@ -289,7 +300,7 @@ impl Collector { ("avg_query_time", 0), ("avg_xact_count", 0), ("avg_sent", 0), - ("avg_received", 0), + ("avg_recv", 0), ("avg_wait_time", 0), ("maxwait_us", 0), ("maxwait", 0), @@ -493,10 +504,14 @@ impl Collector { "avg_query_count", "avgxact_count", "avg_sent", - "avg_received", + "avg_recv", "avg_wait_time", ] { - let total_name = stat.replace("avg_", "total_"); + let total_name = match stat { + &"avg_recv" => "total_received".to_string(), // Because PgBouncer is saving bytes + _ => stat.replace("avg_", "total_"), + }; + let old_value = old_stats.entry(total_name.clone()).or_insert(0); let new_value = stats.get(total_name.as_str()).unwrap_or(&0).to_owned(); let avg = (new_value - *old_value) / (STAT_PERIOD as i64 / 1_000); // Avg / second @@ -515,3 +530,8 @@ impl Collector { pub fn get_stats() -> HashMap> { LATEST_STATS.lock().clone() } + +/// Get the statistics reporter used to update stats across the pools/clients. +pub fn get_reporter() -> Reporter { + (*(*REPORTER.load())).clone() +} diff --git a/tests/pgbench/simple.sql b/tests/pgbench/simple.sql index 0a283bae4..ad5e6139f 100644 --- a/tests/pgbench/simple.sql +++ b/tests/pgbench/simple.sql @@ -12,6 +12,8 @@ SET SHARD TO :shard; +SET SERVER ROLE TO 'auto'; + BEGIN; UPDATE pgbench_accounts SET abalance = abalance + :delta WHERE aid = :aid; @@ -26,3 +28,12 @@ INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (:tid, :bid, :a END; +SET SHARDING KEY TO :aid; + +-- Read load balancing +SELECT abalance FROM pgbench_accounts WHERE aid = :aid; + +SET SERVER ROLE TO 'replica'; + +-- Read load balancing +SELECT abalance FROM pgbench_accounts WHERE aid = :aid; diff --git a/tests/sharding/query_routing_test_primary_replica.sql b/tests/sharding/query_routing_test_primary_replica.sql index db05ba7a8..5fe3cbe3b 100644 --- a/tests/sharding/query_routing_test_primary_replica.sql +++ b/tests/sharding/query_routing_test_primary_replica.sql @@ -151,3 +151,12 @@ SELECT 1; set server role to 'replica'; SeT SeRver Role TO 'PrImARY'; select 1; + +SET PRIMARY READS TO 'on'; +SELECT 1; + +SET PRIMARY READS TO 'off'; +SELECT 1; + +SET PRIMARY READS TO 'default'; +SELECT 1; From 9896964ac6b86e8dbb500608c1888cf984526bea Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 24 Jun 2022 15:14:31 -0700 Subject: [PATCH 143/551] Fix panic & query router bug (#85) * Fix query router bug * Fix panic --- src/client.rs | 6 +++++- src/query_router.rs | 8 +++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client.rs b/src/client.rs index b53bd3352..ea1bf0194 100644 --- a/src/client.rs +++ b/src/client.rs @@ -302,7 +302,11 @@ impl Client { // Handle all custom protocol commands, if any. match query_router.try_execute_command(message.clone()) { // Normal query, not a custom command. - None => (), + None => { + if query_router.query_parser_enabled() { + query_router.infer_role(message.clone()); + } + } // SET SHARD TO Some((Command::SetShard, _)) => { diff --git a/src/query_router.rs b/src/query_router.rs index 995a25c2b..98a47702b 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -126,11 +126,7 @@ impl QueryRouter { // This is not a custom query, try to infer which // server it'll go to if the query parser is enabled. if matches.len() != 1 { - debug!("Regular query"); - if self.query_parser_enabled && self.role() == None { - debug!("Inferring role"); - self.infer_role(buf.clone()); - } + debug!("Regular query, not a command"); return None; } @@ -266,6 +262,8 @@ impl QueryRouter { /// Try to infer which server to connect to based on the contents of the query. pub fn infer_role(&mut self, mut buf: BytesMut) -> bool { + debug!("Inferring role"); + let code = buf.get_u8() as char; let len = buf.get_i32() as usize; From f4f26d9b34e6a4f67e95a4e283b1fe51e47c77f4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 25 Jun 2022 11:46:20 -0700 Subject: [PATCH 144/551] Automatically reload config every seconds (disabled by default) (#86) * Automatically reload config every seconds (disabld by default) * add that --- .circleci/pgcat.toml | 3 +++ Cargo.lock | 4 ++-- pgcat.toml | 3 +++ src/config.rs | 19 ++++++++++++------- src/main.rs | 21 +++++++++++++++++++++ 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index ba3b87515..d7249f152 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -29,6 +29,9 @@ healthcheck_timeout = 100 # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # Seconds +# +autoreload = true + # # User to use for authentication against the server. [user] diff --git a/Cargo.lock b/Cargo.lock index 87d11be7b..b8b4bfd9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.117" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e74d72e0f9b65b5b4ca49a346af3976df0f9c61d550727f349ecd559f251a26c" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "lock_api" diff --git a/pgcat.toml b/pgcat.toml index 435dda95c..70b2fae0e 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -29,6 +29,9 @@ healthcheck_timeout = 1000 # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # Seconds +# Reload config automatically if it changes. +autoreload = false + # # User to use for authentication against the server. [user] diff --git a/src/config.rs b/src/config.rs index 96d5a778f..f6fa129cd 100644 --- a/src/config.rs +++ b/src/config.rs @@ -101,7 +101,7 @@ impl Default for User { } /// General configuration. -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, PartialEq)] pub struct General { pub host: String, pub port: i16, @@ -110,6 +110,7 @@ pub struct General { pub connect_timeout: u64, pub healthcheck_timeout: u64, pub ban_time: i64, + pub autoreload: bool, } impl Default for General { @@ -122,6 +123,7 @@ impl Default for General { connect_timeout: 5000, healthcheck_timeout: 1000, ban_time: 60, + autoreload: false, } } } @@ -143,7 +145,7 @@ impl Default for Shard { } /// Query Router configuration. -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, PartialEq)] pub struct QueryRouter { pub default_role: String, pub query_parser_enabled: bool, @@ -167,7 +169,7 @@ fn default_path() -> String { } /// Configuration wrapper. -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, PartialEq)] pub struct Config { #[serde(default = "default_path")] pub path: String, @@ -374,7 +376,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { Ok(()) } -pub async fn reload_config(client_server_map: ClientServerMap) -> Result<(), Error> { +pub async fn reload_config(client_server_map: ClientServerMap) -> Result { let old_config = get_config(); match parse(&old_config.path).await { @@ -387,11 +389,14 @@ pub async fn reload_config(client_server_map: ClientServerMap) -> Result<(), Err let new_config = get_config(); - if old_config.shards != new_config.shards { + if old_config.shards != new_config.shards || old_config.user != new_config.user { info!("Sharding configuration changed, re-creating server pools"); - ConnectionPool::from_config(client_server_map).await + ConnectionPool::from_config(client_server_map).await?; + Ok(true) + } else if old_config != new_config { + Ok(true) } else { - Ok(()) + Ok(false) } } diff --git a/src/main.rs b/src/main.rs index 70094d883..93715f456 100644 --- a/src/main.rs +++ b/src/main.rs @@ -127,6 +127,7 @@ async fn main() { // Save these for reloading let reload_client_server_map = client_server_map.clone(); + let autoreload_client_server_map = client_server_map.clone(); let addresses = pool.databases(); tokio::task::spawn(async move { @@ -203,6 +204,26 @@ async fn main() { } }); + if config.general.autoreload { + let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); + + tokio::task::spawn(async move { + info!("Config autoreloader started"); + + loop { + interval.tick().await; + match reload_config(autoreload_client_server_map.clone()).await { + Ok(changed) => { + if changed { + get_config().show() + } + } + Err(_) => (), + }; + } + }); + } + // Exit on Ctrl-C (SIGINT) and SIGTERM. let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); From 2bffedad911cb7d9306e60520249c55c101c590b Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 25 Jun 2022 12:22:46 -0700 Subject: [PATCH 145/551] Fix stats dymanic reload (#87) --- src/main.rs | 9 ++------- src/stats.rs | 6 +++++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main.rs b/src/main.rs index 93715f456..7b78e5b14 100644 --- a/src/main.rs +++ b/src/main.rs @@ -60,7 +60,7 @@ mod sharding; mod stats; use config::{get_config, reload_config}; -use pool::{get_pool, ClientServerMap, ConnectionPool}; +use pool::{ClientServerMap, ConnectionPool}; use stats::{Collector, Reporter, REPORTER}; #[tokio::main(worker_threads = 4)] @@ -120,8 +120,6 @@ async fn main() { } }; - let pool = get_pool(); - // Statistics collector task. let collector_tx = tx.clone(); @@ -129,16 +127,13 @@ async fn main() { let reload_client_server_map = client_server_map.clone(); let autoreload_client_server_map = client_server_map.clone(); - let addresses = pool.databases(); tokio::task::spawn(async move { let mut stats_collector = Collector::new(rx, collector_tx); - stats_collector.collect(addresses).await; + stats_collector.collect().await; }); info!("Waiting for clients"); - drop(pool); - // Client connection loop. tokio::task::spawn(async move { loop { diff --git a/src/stats.rs b/src/stats.rs index 7454c583f..59a03dc71 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -6,6 +6,8 @@ use parking_lot::Mutex; use std::collections::HashMap; use tokio::sync::mpsc::{channel, Receiver, Sender}; +use crate::pool::get_pool; + pub static REPORTER: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Reporter::default())); @@ -285,7 +287,7 @@ impl Collector { /// The statistics collection handler. It will collect statistics /// for `address_id`s starting at 0 up to `addresses`. - pub async fn collect(&mut self, addresses: usize) { + pub async fn collect(&mut self) { info!("Events reporter started"); let stats_template = HashMap::from([ @@ -329,6 +331,7 @@ impl Collector { tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD / 15)); loop { interval.tick().await; + let addresses = get_pool().databases(); for address_id in 0..addresses { let _ = tx.try_send(Event { name: EventName::UpdateStats, @@ -346,6 +349,7 @@ impl Collector { tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD)); loop { interval.tick().await; + let addresses = get_pool().databases(); for address_id in 0..addresses { let _ = tx.try_send(Event { name: EventName::UpdateAverages, From b2a5fa7d85984dcc2ea0202c3daca4b59a4a0a16 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 27 Jun 2022 09:46:33 -0700 Subject: [PATCH 146/551] check --- Cargo.lock | 172 ++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 + pgcat.toml | 4 ++ src/client.rs | 12 ++-- src/config.rs | 4 ++ src/errors.rs | 1 + src/main.rs | 3 + src/messages.rs | 42 +++++++----- src/stream.rs | 160 ++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 378 insertions(+), 22 deletions(-) create mode 100644 src/stream.rs diff --git a/Cargo.lock b/Cargo.lock index b8b4bfd9d..cfaa0fbdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,12 +79,24 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" + [[package]] name = "bytes" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + [[package]] name = "cfg-if" version = "1.0.0" @@ -236,6 +248,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "js-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.126" @@ -385,6 +412,7 @@ dependencies = [ "parking_lot", "rand", "regex", + "rustls-pemfile", "serde", "serde_derive", "sha-1", @@ -392,6 +420,7 @@ dependencies = [ "sqlparser", "stringprep", "tokio", + "tokio-rustls", "toml", ] @@ -497,12 +526,58 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustls" +version = "0.20.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +dependencies = [ + "base64", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "serde" version = "1.0.136" @@ -563,6 +638,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "sqlparser" version = "0.14.0" @@ -664,6 +745,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "toml" version = "0.5.8" @@ -700,6 +792,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "version_check" version = "0.9.4" @@ -712,6 +810,80 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasm-bindgen" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" + +[[package]] +name = "web-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index fa63c0e8a..b193e778c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,3 +29,5 @@ hmac = "0.12" sha2 = "0.10" base64 = "0.13" stringprep = "0.1" +tokio-rustls = "*" +rustls-pemfile = "*" diff --git a/pgcat.toml b/pgcat.toml index 70b2fae0e..b2a6b7662 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -32,6 +32,10 @@ ban_time = 60 # Seconds # Reload config automatically if it changes. autoreload = false +# TLS +tls_certificate = "server.cert" +tls_private_key = "server.key" + # # User to use for authentication against the server. [user] diff --git a/src/client.rs b/src/client.rs index ea1bf0194..c02fcf8e4 100644 --- a/src/client.rs +++ b/src/client.rs @@ -19,13 +19,13 @@ use crate::server::Server; use crate::stats::{get_reporter, Reporter}; /// The client state. One of these is created per client. -pub struct Client { +pub struct Client { /// The reads are buffered (8K by default). - read: BufReader, + read: BufReader, /// We buffer the writes ourselves because we know the protocol /// better than a stock buffer. - write: OwnedWriteHalf, + write: S, /// Internal buffer, where we place messages until we have to flush /// them to the backend. @@ -63,13 +63,13 @@ pub struct Client { last_server_id: Option, } -impl Client { +impl Client { /// Perform client startup sequence. /// See docs: pub async fn startup( mut stream: TcpStream, client_server_map: ClientServerMap, - ) -> Result { + ) -> Result, Error> { let config = get_config(); let transaction_mode = config.general.pool_mode == "transaction"; let stats = get_reporter(); @@ -650,7 +650,7 @@ impl Client { } } -impl Drop for Client { +impl Drop for Client { fn drop(&mut self) { // Update statistics. if let Some(address_id) = self.last_address_id { diff --git a/src/config.rs b/src/config.rs index f6fa129cd..170054184 100644 --- a/src/config.rs +++ b/src/config.rs @@ -111,6 +111,8 @@ pub struct General { pub healthcheck_timeout: u64, pub ban_time: i64, pub autoreload: bool, + pub tls_certificate: Option, + pub tls_private_key: Option, } impl Default for General { @@ -124,6 +126,8 @@ impl Default for General { healthcheck_timeout: 1000, ban_time: 60, autoreload: false, + tls_certificate: None, + tls_private_key: None, } } } diff --git a/src/errors.rs b/src/errors.rs index b07d50883..cc8f65d00 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -10,4 +10,5 @@ pub enum Error { BadConfig, AllServersDown, ClientError, + TlsError, } diff --git a/src/main.rs b/src/main.rs index 7b78e5b14..05bfb2376 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,6 +33,8 @@ extern crate serde_derive; extern crate sqlparser; extern crate tokio; extern crate toml; +extern crate tokio_rustls; +extern crate rustls_pemfile; use log::{debug, error, info}; use parking_lot::Mutex; @@ -58,6 +60,7 @@ mod scram; mod server; mod sharding; mod stats; +mod stream; use config::{get_config, reload_config}; use pool::{ClientServerMap, ConnectionPool}; diff --git a/src/messages.rs b/src/messages.rs index 993545bbb..960a3b6f2 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -30,7 +30,8 @@ impl From<&DataType> for i32 { } /// Tell the client that authentication handshake completed successfully. -pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { +pub async fn auth_ok(stream: &mut S) -> Result<(), Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { let mut auth_ok = BytesMut::with_capacity(9); auth_ok.put_u8(b'R'); @@ -41,7 +42,8 @@ pub async fn auth_ok(stream: &mut TcpStream) -> Result<(), Error> { } /// Generate md5 password challenge. -pub async fn md5_challenge(stream: &mut TcpStream) -> Result<[u8; 4], Error> { +pub async fn md5_challenge(stream: &mut S) -> Result<[u8; 4], Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { // let mut rng = rand::thread_rng(); let salt: [u8; 4] = [ rand::random(), @@ -62,11 +64,12 @@ pub async fn md5_challenge(stream: &mut TcpStream) -> Result<[u8; 4], Error> { /// Give the client the process_id and secret we generated /// used in query cancellation. -pub async fn backend_key_data( - stream: &mut TcpStream, +pub async fn backend_key_data( + stream: &mut S, backend_id: i32, secret_key: i32, -) -> Result<(), Error> { +) -> Result<(), Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { let mut key_data = BytesMut::from(&b"K"[..]); key_data.put_i32(12); key_data.put_i32(backend_id); @@ -87,7 +90,8 @@ pub fn simple_query(query: &str) -> BytesMut { } /// Tell the client we're ready for another query. -pub async fn ready_for_query(stream: &mut TcpStream) -> Result<(), Error> { +pub async fn ready_for_query(stream: &mut S) -> Result<(), Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { let mut bytes = BytesMut::with_capacity(5); bytes.put_u8(b'Z'); @@ -205,12 +209,13 @@ pub fn md5_hash_password(user: &str, password: &str, salt: &[u8]) -> Vec { /// Send password challenge response to the server. /// This is the MD5 challenge. -pub async fn md5_password( - stream: &mut TcpStream, +pub async fn md5_password( + stream: &mut S, user: &str, password: &str, salt: &[u8], -) -> Result<(), Error> { +) -> Result<(), Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { let password = md5_hash_password(user, password, salt); let mut message = BytesMut::with_capacity(password.len() as usize + 5); @@ -225,10 +230,11 @@ pub async fn md5_password( /// Implements a response to our custom `SET SHARDING KEY` /// and `SET SERVER ROLE` commands. /// This tells the client we're ready for the next query. -pub async fn custom_protocol_response_ok( - stream: &mut OwnedWriteHalf, +pub async fn custom_protocol_response_ok( + stream: &mut S, message: &str, -) -> Result<(), Error> { +) -> Result<(), Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { let mut res = BytesMut::with_capacity(25); let set_complete = BytesMut::from(&format!("{}\0", message)[..]); @@ -250,7 +256,8 @@ pub async fn custom_protocol_response_ok( /// Send a custom error message to the client. /// Tell the client we are ready for the next query and no rollback is necessary. /// Docs on error codes: . -pub async fn error_response(stream: &mut OwnedWriteHalf, message: &str) -> Result<(), Error> { +pub async fn error_response(stream: &mut S, message: &str) -> Result<(), Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { let mut error = BytesMut::new(); // Error level @@ -291,7 +298,8 @@ pub async fn error_response(stream: &mut OwnedWriteHalf, message: &str) -> Resul Ok(write_all_half(stream, res).await?) } -pub async fn wrong_password(stream: &mut TcpStream, user: &str) -> Result<(), Error> { +pub async fn wrong_password(stream: &mut S, user: &str) -> Result<(), Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { let mut error = BytesMut::new(); // Error level @@ -430,7 +438,8 @@ pub fn command_complete(command: &str) -> BytesMut { } /// Write all data in the buffer to the TcpStream. -pub async fn write_all(stream: &mut TcpStream, buf: BytesMut) -> Result<(), Error> { +pub async fn write_all(stream: &mut S, buf: BytesMut) -> Result<(), Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { match stream.write_all(&buf).await { Ok(_) => Ok(()), Err(_) => return Err(Error::SocketError), @@ -438,7 +447,8 @@ pub async fn write_all(stream: &mut TcpStream, buf: BytesMut) -> Result<(), Erro } /// Write all the data in the buffer to the TcpStream, write owned half (see mpsc). -pub async fn write_all_half(stream: &mut OwnedWriteHalf, buf: BytesMut) -> Result<(), Error> { +pub async fn write_all_half(stream: &mut S, buf: BytesMut) -> Result<(), Error> +where S: tokio::io::AsyncWrite + std::marker::Unpin { match stream.write_all(&buf).await { Ok(_) => Ok(()), Err(_) => return Err(Error::SocketError), diff --git a/src/stream.rs b/src/stream.rs new file mode 100644 index 000000000..41d3cd1bf --- /dev/null +++ b/src/stream.rs @@ -0,0 +1,160 @@ +// Stream wrapper. + +use bytes::{Buf, BufMut, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader, split, ReadHalf, WriteHalf}; +use tokio::net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, +}; +use tokio_rustls::server::TlsStream; +use rustls_pemfile::{certs, rsa_private_keys}; +use tokio_rustls::rustls::{self, Certificate, PrivateKey}; +use tokio_rustls::TlsAcceptor; +use std::sync::Arc; +use std::path::Path; + +use crate::config::get_config; +use crate::errors::Error; + +// TLS +fn load_certs(path: &std::path::Path) -> std::io::Result> { + certs(&mut std::io::BufReader::new(std::fs::File::open(path)?)) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid cert")) + .map(|mut certs| certs.drain(..).map(Certificate).collect()) +} + +fn load_keys(path: &std::path::Path) -> std::io::Result> { + rsa_private_keys(&mut std::io::BufReader::new(std::fs::File::open(path)?)) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid key")) + .map(|mut keys| keys.drain(..).map(PrivateKey).collect()) +} + +struct Tls { + acceptor: TlsAcceptor, +} + +impl Tls { + pub fn new() -> Result { + let config = get_config(); + + let certs = match load_certs(&Path::new(&config.general.tls_certificate.unwrap())) { + Ok(certs) => certs, + Err(_) => return Err(Error::TlsError), + }; + + let mut keys = match load_keys(&Path::new(&config.general.tls_private_key.unwrap())) { + Ok(keys) => keys, + Err(_) => return Err(Error::TlsError), + }; + + let config = match rustls::ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(certs, keys.remove(0)) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err)) { + Ok(c) => c, + Err(_) => return Err(Error::TlsError) + }; + + Ok(Tls { + acceptor: TlsAcceptor::from(Arc::new(config)), + }) + } +} + +struct Stream { + read: Option>, + write: Option, + tls_read: Option>>>, + tls_write: Option>>, +} + + +impl Stream { + pub async fn new(stream: TcpStream, tls: Option) -> Result { + + let config = get_config(); + + match tls { + None => { + let (read, write) = stream.into_split(); + let read = BufReader::new(read); + Ok( + Self { + read: Some(read), + write: Some(write), + tls_read: None, + tls_write: None, + } + ) + } + + Some(tls) => { + let mut tls_stream = match tls.acceptor.accept(stream).await { + Ok(stream) => stream, + Err(_) => return Err(Error::TlsError), + }; + + let (read, write) = split(tls_stream); + + Ok(Self{ + read: None, + write: None, + tls_read: Some(BufReader::new(read)), + tls_write: Some(write), + }) + } + } + } + + async fn read(stream: &mut S) -> Result + where S: tokio::io::AsyncRead + std::marker::Unpin { + + let code = match stream.read_u8().await { + Ok(code) => code, + Err(_) => return Err(Error::SocketError), + }; + + let len = match stream.read_i32().await { + Ok(len) => len, + Err(_) => return Err(Error::SocketError), + }; + + let mut buf = vec![0u8; len as usize - 4]; + + match stream.read_exact(&mut buf).await { + Ok(_) => (), + Err(_) => return Err(Error::SocketError), + }; + + let mut bytes = BytesMut::with_capacity(len as usize + 1); + + bytes.put_u8(code); + bytes.put_i32(len); + bytes.put_slice(&buf); + + Ok(bytes) + } + + async fn write(stream: &mut S, buf: &BytesMut) -> Result<(), Error> + where S: tokio::io::AsyncWrite + std::marker::Unpin { + match stream.write_all(buf).await { + Ok(_) => Ok(()), + Err(_) => return Err(Error::SocketError), + } + } + + pub async fn read_message(&mut self) -> Result { + match &self.read { + Some(read) => Self::read(self.read.as_mut().unwrap()).await, + None => Self::read(self.tls_read.as_mut().unwrap()).await, + } + } + + pub async fn write_all(&mut self, buf: &BytesMut) -> Result<(), Error> { + match &self.write { + Some(write) => Self::write(self.write.as_mut().unwrap(), buf).await, + None => Self::write(self.tls_write.as_mut().unwrap(), buf).await, + } + } +} \ No newline at end of file From 254eb7fe744ddfee3d6de2010fcbcba56363a8ba Mon Sep 17 00:00:00 2001 From: Lev Date: Mon, 27 Jun 2022 15:52:01 -0700 Subject: [PATCH 147/551] at least it compiles --- src/admin.rs | 54 ++++-- src/client.rs | 484 +++++++++++++++++++++++++++++++++--------------- src/main.rs | 67 ++++--- src/messages.rs | 59 ++++-- src/stream.rs | 201 ++++++++------------ 5 files changed, 540 insertions(+), 325 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index b7a5b6fd8..622c14a97 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -2,7 +2,7 @@ use bytes::{Buf, BufMut, BytesMut}; use log::{info, trace}; use std::collections::HashMap; -use tokio::net::tcp::OwnedWriteHalf; +// use tokio::net::tcp::T; use crate::config::{get_config, reload_config}; use crate::errors::Error; @@ -12,12 +12,15 @@ use crate::stats::get_stats; use crate::ClientServerMap; /// Handle admin client. -pub async fn handle_admin( - stream: &mut OwnedWriteHalf, +pub async fn handle_admin( + stream: &mut T, mut query: BytesMut, pool: ConnectionPool, client_server_map: ClientServerMap, -) -> Result<(), Error> { +) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ let code = query.get_u8() as char; if code != 'Q' { @@ -61,7 +64,10 @@ pub async fn handle_admin( } /// Column-oriented statistics. -async fn show_lists(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { +async fn show_lists(stream: &mut T, pool: &ConnectionPool) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ let stats = get_stats(); let columns = vec![("list", DataType::Text), ("items", DataType::Int4)]; @@ -128,7 +134,10 @@ async fn show_lists(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul } /// Show PgCat version. -async fn show_version(stream: &mut OwnedWriteHalf) -> Result<(), Error> { +async fn show_version(stream: &mut T) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ let mut res = BytesMut::new(); res.put(row_description(&vec![("version", DataType::Text)])); @@ -143,7 +152,10 @@ async fn show_version(stream: &mut OwnedWriteHalf) -> Result<(), Error> { } /// Show utilization of connection pools for each shard and replicas. -async fn show_pools(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { +async fn show_pools(stream: &mut T, pool: &ConnectionPool) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ let stats = get_stats(); let config = get_config(); @@ -197,7 +209,10 @@ async fn show_pools(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Resul } /// Show shards and replicas. -async fn show_databases(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { +async fn show_databases(stream: &mut T, pool: &ConnectionPool) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ let config = get_config(); // Columns @@ -258,15 +273,18 @@ async fn show_databases(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> R /// Ignore any SET commands the client sends. /// This is common initialization done by ORMs. -async fn ignore_set(stream: &mut OwnedWriteHalf) -> Result<(), Error> { +async fn ignore_set(stream: &mut T) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ custom_protocol_response_ok(stream, "SET").await } /// Reload the configuration file without restarting the process. -async fn reload( - stream: &mut OwnedWriteHalf, - client_server_map: ClientServerMap, -) -> Result<(), Error> { +async fn reload(stream: &mut T, client_server_map: ClientServerMap) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ info!("Reloading config"); reload_config(client_server_map).await?; @@ -286,7 +304,10 @@ async fn reload( } /// Shows current configuration. -async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { +async fn show_config(stream: &mut T) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ let config = &get_config(); let config: HashMap = config.into(); @@ -329,7 +350,10 @@ async fn show_config(stream: &mut OwnedWriteHalf) -> Result<(), Error> { } /// Show shard and replicas statistics. -async fn show_stats(stream: &mut OwnedWriteHalf, pool: &ConnectionPool) -> Result<(), Error> { +async fn show_stats(stream: &mut T, pool: &ConnectionPool) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ let columns = vec![ ("database", DataType::Text), ("total_xact_count", DataType::Numeric), diff --git a/src/client.rs b/src/client.rs index c02fcf8e4..4e8ba736a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,7 +2,7 @@ use bytes::{Buf, BufMut, BytesMut}; use log::{debug, error, trace}; use std::collections::HashMap; -use tokio::io::{AsyncReadExt, BufReader}; +use tokio::io::{split, AsyncReadExt, AsyncWriteExt, BufReader, ReadHalf, WriteHalf}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, TcpStream, @@ -17,15 +17,25 @@ use crate::pool::{get_pool, ClientServerMap}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; use crate::stats::{get_reporter, Reporter}; +use crate::stream::Tls; + +use tokio_rustls::server::TlsStream; + +/// Type of connection received from client. +enum ClientConnectionType { + Startup, + Tls, + CancelQuery, +} /// The client state. One of these is created per client. -pub struct Client { +pub struct Client { /// The reads are buffered (8K by default). - read: BufReader, + read: BufReader, /// We buffer the writes ourselves because we know the protocol /// better than a stock buffer. - write: S, + write: T, /// Internal buffer, where we place messages until we have to flush /// them to the backend. @@ -63,163 +73,347 @@ pub struct Client { last_server_id: Option, } -impl Client { - /// Perform client startup sequence. - /// See docs: - pub async fn startup( - mut stream: TcpStream, - client_server_map: ClientServerMap, - ) -> Result, Error> { - let config = get_config(); - let transaction_mode = config.general.pool_mode == "transaction"; - let stats = get_reporter(); - - loop { - trace!("Waiting for StartupMessage"); - - // Could be StartupMessage, SSLRequest or CancelRequest. - let len = match stream.read_i32().await { - Ok(len) => len, - Err(_) => return Err(Error::ClientBadStartup), - }; - - let mut startup = vec![0u8; len as usize - 4]; - - match stream.read_exact(&mut startup).await { - Ok(_) => (), - Err(_) => return Err(Error::ClientBadStartup), - }; - - let mut bytes = BytesMut::from(&startup[..]); - let code = bytes.get_i32(); +/// Main client loop. +pub async fn client_loop( + mut stream: TcpStream, + client_server_map: ClientServerMap, +) -> Result<(), Error> { + match get_startup::(&mut stream).await { + Ok((ClientConnectionType::Tls, bytes)) => { + match startup_tls(stream, client_server_map).await { + Ok(mut client) => client.handle().await, + Err(err) => Err(err), + } + } - match code { - // Client wants SSL. We don't support it at the moment. - SSL_REQUEST_CODE => { - trace!("Rejecting SSLRequest"); + Ok((ClientConnectionType::Startup, bytes)) => { + let (read, write) = split(stream); + match Client::handle_startup(read, write, bytes, client_server_map).await { + Ok(mut client) => client.handle().await, + Err(err) => Err(err), + } + } - let mut no = BytesMut::with_capacity(1); - no.put_u8(b'N'); + Ok((ClientConnectionType::CancelQuery, bytes)) => { + return Err(Error::ProtocolSyncError); + } - write_all(&mut stream, no).await?; - } + Err(err) => Err(err), + } +} - // Regular startup message. - PROTOCOL_VERSION_NUMBER => { - trace!("Got StartupMessage"); - let parameters = parse_startup(bytes.clone())?; +async fn get_startup(stream: &mut S) -> Result<(ClientConnectionType, BytesMut), Error> +where + S: tokio::io::AsyncRead + std::marker::Unpin + tokio::io::AsyncWrite, +{ + // Get startup message length. + let len = match stream.read_i32().await { + Ok(len) => len, + Err(_) => return Err(Error::ClientBadStartup), + }; + + // Get the rest of the message. + let mut startup = vec![0u8; len as usize - 4]; + match stream.read_exact(&mut startup).await { + Ok(_) => (), + Err(_) => return Err(Error::ClientBadStartup), + }; + + let mut bytes = BytesMut::from(&startup[..]); + let code = bytes.get_i32(); + + match code { + // Client is requesting SSL (TLS). + SSL_REQUEST_CODE => Ok((ClientConnectionType::Tls, bytes)), + + // Client wants to use plain text, requesting regular startup. + PROTOCOL_VERSION_NUMBER => Ok((ClientConnectionType::Startup, bytes)), + + // Client is requesting to cancel a running query (plain text connection). + CANCEL_REQUEST_CODE => Ok((ClientConnectionType::CancelQuery, bytes)), + _ => Err(Error::ProtocolSyncError), + } +} - // Generate random backend ID and secret key - let process_id: i32 = rand::random(); - let secret_key: i32 = rand::random(); +/// Handle TLS connection negotation. +pub async fn startup_tls( + mut stream: TcpStream, + client_server_map: ClientServerMap, +) -> Result>, WriteHalf>>, Error> { + // Accept SSL request if SSL is configured. + let mut yes = BytesMut::new(); + yes.put_u8(b'S'); + write_all(&mut stream, yes).await?; + + // Negotiate TLS. + let mut tls = Tls::new().unwrap(); + let mut stream = match tls.acceptor.accept(stream).await { + Ok(stream) => stream, + Err(_) => return Err(Error::TlsError), + }; + + match get_startup::>(&mut stream).await { + Ok((ClientConnectionType::Startup, bytes)) => { + let (read, write) = split(stream); + Client::handle_startup(read, write, bytes, client_server_map).await + } + _ => Err(Error::ProtocolSyncError), + } +} - // Perform MD5 authentication. - // TODO: Add SASL support. - let salt = md5_challenge(&mut stream).await?; +impl Client +where + S: tokio::io::AsyncRead + std::marker::Unpin, + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + // Perform client startup sequence in TLS. + pub async fn handle_startup( + mut read: S, + mut write: T, + bytes: BytesMut, // The rest of the startup message. + client_server_map: ClientServerMap, + ) -> Result, Error> { + let config = get_config(); + let transaction_mode = config.general.pool_mode == "transaction"; + let stats = get_reporter(); - let code = match stream.read_u8().await { - Ok(p) => p, - Err(_) => return Err(Error::SocketError), - }; + trace!("Got StartupMessage"); + let parameters = parse_startup(bytes.clone())?; - // PasswordMessage - if code as char != 'p' { - debug!("Expected p, got {}", code as char); - return Err(Error::ProtocolSyncError); - } + // Generate random backend ID and secret key + let process_id: i32 = rand::random(); + let secret_key: i32 = rand::random(); - let len = match stream.read_i32().await { - Ok(len) => len, - Err(_) => return Err(Error::SocketError), - }; + // Perform MD5 authentication. + // TODO: Add SASL support. + let salt = md5_challenge(&mut write).await?; - let mut password_response = vec![0u8; (len - 4) as usize]; + let code = match read.read_u8().await { + Ok(p) => p, + Err(_) => return Err(Error::SocketError), + }; - match stream.read_exact(&mut password_response).await { - Ok(_) => (), - Err(_) => return Err(Error::SocketError), - }; + // PasswordMessage + if code as char != 'p' { + debug!("Expected p, got {}", code as char); + return Err(Error::ProtocolSyncError); + } - // Compare server and client hashes. - let password_hash = - md5_hash_password(&config.user.name, &config.user.password, &salt); + let len = match read.read_i32().await { + Ok(len) => len, + Err(_) => return Err(Error::SocketError), + }; - if password_hash != password_response { - debug!("Password authentication failed"); - wrong_password(&mut stream, &config.user.name).await?; - return Err(Error::ClientError); - } + let mut password_response = vec![0u8; (len - 4) as usize]; - debug!("Password authentication successful"); - - auth_ok(&mut stream).await?; - write_all(&mut stream, get_pool().server_info()).await?; - backend_key_data(&mut stream, process_id, secret_key).await?; - ready_for_query(&mut stream).await?; - - trace!("Startup OK"); - - let database = parameters - .get("database") - .unwrap_or(parameters.get("user").unwrap()); - let admin = ["pgcat", "pgbouncer"] - .iter() - .filter(|db| *db == &database) - .count() - == 1; - - // Split the read and write streams - // so we can control buffering. - let (read, write) = stream.into_split(); - - return Ok(Client { - read: BufReader::new(read), - write: write, - buffer: BytesMut::with_capacity(8196), - cancel_mode: false, - transaction_mode: transaction_mode, - process_id: process_id, - secret_key: secret_key, - client_server_map: client_server_map, - parameters: parameters, - stats: stats, - admin: admin, - last_address_id: None, - last_server_id: None, - }); - } + match read.read_exact(&mut password_response).await { + Ok(_) => (), + Err(_) => return Err(Error::SocketError), + }; - // Query cancel request. - CANCEL_REQUEST_CODE => { - let (read, write) = stream.into_split(); - - let process_id = bytes.get_i32(); - let secret_key = bytes.get_i32(); - - return Ok(Client { - read: BufReader::new(read), - write: write, - buffer: BytesMut::with_capacity(8196), - cancel_mode: true, - transaction_mode: transaction_mode, - process_id: process_id, - secret_key: secret_key, - client_server_map: client_server_map, - parameters: HashMap::new(), - stats: stats, - admin: false, - last_address_id: None, - last_server_id: None, - }); - } + // Compare server and client hashes. + let password_hash = md5_hash_password(&config.user.name, &config.user.password, &salt); - _ => { - return Err(Error::ProtocolSyncError); - } - }; + if password_hash != password_response { + debug!("Password authentication failed"); + wrong_password(&mut write, &config.user.name).await?; + return Err(Error::ClientError); } + + debug!("Password authentication successful"); + + auth_ok(&mut write).await?; + write_all(&mut write, get_pool().server_info()).await?; + backend_key_data(&mut write, process_id, secret_key).await?; + ready_for_query(&mut write).await?; + + trace!("Startup OK"); + + let database = parameters + .get("database") + .unwrap_or(parameters.get("user").unwrap()); + let admin = ["pgcat", "pgbouncer"] + .iter() + .filter(|db| *db == &database) + .count() + == 1; + + // Split the read and write streams + // so we can control buffering. + + return Ok(Client { + read: BufReader::new(read), + write: write, + buffer: BytesMut::with_capacity(8196), + cancel_mode: false, + transaction_mode: transaction_mode, + process_id: process_id, + secret_key: secret_key, + client_server_map: client_server_map, + parameters: parameters, + stats: stats, + admin: admin, + last_address_id: None, + last_server_id: None, + }); } + /// Perform client startup sequence. + /// See docs: + // pub async fn startup( + // mut stream: TcpStream, + // client_server_map: ClientServerMap, + // ) -> Result, WriteHalf>, Error> { + // let config = get_config(); + // let transaction_mode = config.general.pool_mode == "transaction"; + // let stats = get_reporter(); + + // loop { + // trace!("Waiting for StartupMessage"); + + // // Could be StartupMessage, SSLRequest or CancelRequest. + // let len = match stream.read_i32().await { + // Ok(len) => len, + // Err(_) => return Err(Error::ClientBadStartup), + // }; + + // let mut startup = vec![0u8; len as usize - 4]; + + // match stream.read_exact(&mut startup).await { + // Ok(_) => (), + // Err(_) => return Err(Error::ClientBadStartup), + // }; + + // let mut bytes = BytesMut::from(&startup[..]); + // let code = bytes.get_i32(); + + // match code { + // // Client wants SSL. We don't support it at the moment. + // SSL_REQUEST_CODE => { + // trace!("Rejecting SSLRequest"); + + // let mut no = BytesMut::with_capacity(1); + // no.put_u8(b'N'); + + // write_all(&mut stream, no).await?; + // } + + // // Regular startup message. + // PROTOCOL_VERSION_NUMBER => { + // trace!("Got StartupMessage"); + // let parameters = parse_startup(bytes.clone())?; + + // // Generate random backend ID and secret key + // let process_id: i32 = rand::random(); + // let secret_key: i32 = rand::random(); + + // // Perform MD5 authentication. + // // TODO: Add SASL support. + // let salt = md5_challenge(&mut stream).await?; + + // let code = match stream.read_u8().await { + // Ok(p) => p, + // Err(_) => return Err(Error::SocketError), + // }; + + // // PasswordMessage + // if code as char != 'p' { + // debug!("Expected p, got {}", code as char); + // return Err(Error::ProtocolSyncError); + // } + + // let len = match stream.read_i32().await { + // Ok(len) => len, + // Err(_) => return Err(Error::SocketError), + // }; + + // let mut password_response = vec![0u8; (len - 4) as usize]; + + // match stream.read_exact(&mut password_response).await { + // Ok(_) => (), + // Err(_) => return Err(Error::SocketError), + // }; + + // // Compare server and client hashes. + // let password_hash = + // md5_hash_password(&config.user.name, &config.user.password, &salt); + + // if password_hash != password_response { + // debug!("Password authentication failed"); + // wrong_password(&mut stream, &config.user.name).await?; + // return Err(Error::ClientError); + // } + + // debug!("Password authentication successful"); + + // auth_ok(&mut stream).await?; + // write_all(&mut stream, get_pool().server_info()).await?; + // backend_key_data(&mut stream, process_id, secret_key).await?; + // ready_for_query(&mut stream).await?; + + // trace!("Startup OK"); + + // let database = parameters + // .get("database") + // .unwrap_or(parameters.get("user").unwrap()); + // let admin = ["pgcat", "pgbouncer"] + // .iter() + // .filter(|db| *db == &database) + // .count() + // == 1; + + // // Split the read and write streams + // // so we can control buffering. + // let (read, write) = split(stream); + + // return Ok(Client { + // read: BufReader::new(read), + // write: write, + // buffer: BytesMut::with_capacity(8196), + // cancel_mode: false, + // transaction_mode: transaction_mode, + // process_id: process_id, + // secret_key: secret_key, + // client_server_map: client_server_map, + // parameters: parameters, + // stats: stats, + // admin: admin, + // last_address_id: None, + // last_server_id: None, + // }); + // } + + // // Query cancel request. + // CANCEL_REQUEST_CODE => { + // let (read, write) = split(stream); + + // let process_id = bytes.get_i32(); + // let secret_key = bytes.get_i32(); + + // return Ok(Client { + // read: BufReader::new(read), + // write: write, + // buffer: BytesMut::with_capacity(8196), + // cancel_mode: true, + // transaction_mode: transaction_mode, + // process_id: process_id, + // secret_key: secret_key, + // client_server_map: client_server_map, + // parameters: HashMap::new(), + // stats: stats, + // admin: false, + // last_address_id: None, + // last_server_id: None, + // }); + // } + + // _ => { + // return Err(Error::ProtocolSyncError); + // } + // }; + // } + // } + /// Handle a connected and authenticated client. pub async fn handle(&mut self) -> Result<(), Error> { // The client wants to cancel a query it has issued previously. @@ -414,8 +608,8 @@ impl Drop for Client { +impl Drop for Client { fn drop(&mut self) { // Update statistics. if let Some(address_id) = self.last_address_id { diff --git a/src/main.rs b/src/main.rs index 05bfb2376..439c01ac3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,13 +28,13 @@ extern crate log; extern crate md5; extern crate num_cpus; extern crate once_cell; +extern crate rustls_pemfile; extern crate serde; extern crate serde_derive; extern crate sqlparser; extern crate tokio; -extern crate toml; extern crate tokio_rustls; -extern crate rustls_pemfile; +extern crate toml; use log::{debug, error, info}; use parking_lot::Mutex; @@ -45,6 +45,11 @@ use tokio::{ sync::mpsc, }; +use tokio::net::{ + tcp::{OwnedReadHalf, OwnedWriteHalf}, + TcpStream, +}; + use std::collections::HashMap; use std::sync::Arc; @@ -62,6 +67,7 @@ mod sharding; mod stats; mod stream; +use crate::constants::*; use config::{get_config, reload_config}; use pool::{ClientServerMap, ConnectionPool}; use stats::{Collector, Reporter, REPORTER}; @@ -153,32 +159,45 @@ async fn main() { // Handle client. tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); - match client::Client::startup(socket, client_server_map).await { - Ok(mut client) => { - info!("Client {:?} connected", addr); - - match client.handle().await { - Ok(()) => { - let duration = chrono::offset::Utc::now().naive_utc() - start; - - info!( - "Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); - } - - Err(err) => { - error!("Client disconnected with error: {:?}", err); - client.release(); - } - } - } - + // match client::get_startup(&mut socket) { + // Ok((code, bytes)) => match code { + // SSL_REQUEST_CODE => client::Client::tls_startup< + // } + // } + + match client::client_loop(socket, client_server_map).await { + Ok(_) => (), Err(err) => { debug!("Client failed to login: {:?}", err); } }; + + // match client::Client::::startup(socket, client_server_map).await { + // Ok(mut client) => { + // info!("Client {:?} connected", addr); + + // match client.handle().await { + // Ok(()) => { + // let duration = chrono::offset::Utc::now().naive_utc() - start; + + // info!( + // "Client {:?} disconnected, session duration: {}", + // addr, + // format_duration(&duration) + // ); + // } + + // Err(err) => { + // error!("Client disconnected with error: {:?}", err); + // client.release(); + // } + // } + // } + + // Err(err) => { + // debug!("Client failed to login: {:?}", err); + // } + // }; }); } }); diff --git a/src/messages.rs b/src/messages.rs index 960a3b6f2..7b0479299 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -31,7 +31,9 @@ impl From<&DataType> for i32 { /// Tell the client that authentication handshake completed successfully. pub async fn auth_ok(stream: &mut S) -> Result<(), Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ let mut auth_ok = BytesMut::with_capacity(9); auth_ok.put_u8(b'R'); @@ -43,7 +45,9 @@ where S: tokio::io::AsyncWrite + std::marker::Unpin { /// Generate md5 password challenge. pub async fn md5_challenge(stream: &mut S) -> Result<[u8; 4], Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ // let mut rng = rand::thread_rng(); let salt: [u8; 4] = [ rand::random(), @@ -69,7 +73,9 @@ pub async fn backend_key_data( backend_id: i32, secret_key: i32, ) -> Result<(), Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ let mut key_data = BytesMut::from(&b"K"[..]); key_data.put_i32(12); key_data.put_i32(backend_id); @@ -91,7 +97,9 @@ pub fn simple_query(query: &str) -> BytesMut { /// Tell the client we're ready for another query. pub async fn ready_for_query(stream: &mut S) -> Result<(), Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ let mut bytes = BytesMut::with_capacity(5); bytes.put_u8(b'Z'); @@ -215,7 +223,9 @@ pub async fn md5_password( password: &str, salt: &[u8], ) -> Result<(), Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ let password = md5_hash_password(user, password, salt); let mut message = BytesMut::with_capacity(password.len() as usize + 5); @@ -230,11 +240,10 @@ where S: tokio::io::AsyncWrite + std::marker::Unpin { /// Implements a response to our custom `SET SHARDING KEY` /// and `SET SERVER ROLE` commands. /// This tells the client we're ready for the next query. -pub async fn custom_protocol_response_ok( - stream: &mut S, - message: &str, -) -> Result<(), Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +pub async fn custom_protocol_response_ok(stream: &mut S, message: &str) -> Result<(), Error> +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ let mut res = BytesMut::with_capacity(25); let set_complete = BytesMut::from(&format!("{}\0", message)[..]); @@ -257,7 +266,9 @@ where S: tokio::io::AsyncWrite + std::marker::Unpin { /// Tell the client we are ready for the next query and no rollback is necessary. /// Docs on error codes: . pub async fn error_response(stream: &mut S, message: &str) -> Result<(), Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ let mut error = BytesMut::new(); // Error level @@ -299,7 +310,9 @@ where S: tokio::io::AsyncWrite + std::marker::Unpin { } pub async fn wrong_password(stream: &mut S, user: &str) -> Result<(), Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ let mut error = BytesMut::new(); // Error level @@ -333,11 +346,10 @@ where S: tokio::io::AsyncWrite + std::marker::Unpin { } /// Respond to a SHOW SHARD command. -pub async fn show_response( - stream: &mut OwnedWriteHalf, - name: &str, - value: &str, -) -> Result<(), Error> { +pub async fn show_response(stream: &mut S, name: &str, value: &str) -> Result<(), Error> +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ // A SELECT response consists of: // 1. RowDescription // 2. One or more DataRow @@ -439,7 +451,9 @@ pub fn command_complete(command: &str) -> BytesMut { /// Write all data in the buffer to the TcpStream. pub async fn write_all(stream: &mut S, buf: BytesMut) -> Result<(), Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ match stream.write_all(&buf).await { Ok(_) => Ok(()), Err(_) => return Err(Error::SocketError), @@ -448,7 +462,9 @@ where S: tokio::io::AsyncWrite + std::marker::Unpin { /// Write all the data in the buffer to the TcpStream, write owned half (see mpsc). pub async fn write_all_half(stream: &mut S, buf: BytesMut) -> Result<(), Error> -where S: tokio::io::AsyncWrite + std::marker::Unpin { +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ match stream.write_all(&buf).await { Ok(_) => Ok(()), Err(_) => return Err(Error::SocketError), @@ -456,7 +472,10 @@ where S: tokio::io::AsyncWrite + std::marker::Unpin { } /// Read a complete message from the socket. -pub async fn read_message(stream: &mut BufReader) -> Result { +pub async fn read_message(stream: &mut S) -> Result +where + S: tokio::io::AsyncRead + std::marker::Unpin, +{ let code = match stream.read_u8().await { Ok(code) => code, Err(_) => return Err(Error::SocketError), diff --git a/src/stream.rs b/src/stream.rs index 41d3cd1bf..9c19b89ed 100644 --- a/src/stream.rs +++ b/src/stream.rs @@ -1,17 +1,17 @@ // Stream wrapper. use bytes::{Buf, BufMut, BytesMut}; -use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader, split, ReadHalf, WriteHalf}; +use rustls_pemfile::{certs, rsa_private_keys}; +use std::path::Path; +use std::sync::Arc; +use tokio::io::{split, AsyncReadExt, AsyncWriteExt, BufReader, ReadHalf, WriteHalf}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, TcpStream, }; -use tokio_rustls::server::TlsStream; -use rustls_pemfile::{certs, rsa_private_keys}; use tokio_rustls::rustls::{self, Certificate, PrivateKey}; +use tokio_rustls::server::TlsStream; use tokio_rustls::TlsAcceptor; -use std::sync::Arc; -use std::path::Path; use crate::config::get_config; use crate::errors::Error; @@ -29,132 +29,91 @@ fn load_keys(path: &std::path::Path) -> std::io::Result> { .map(|mut keys| keys.drain(..).map(PrivateKey).collect()) } -struct Tls { - acceptor: TlsAcceptor, +pub struct Tls { + pub acceptor: TlsAcceptor, } impl Tls { - pub fn new() -> Result { - let config = get_config(); - - let certs = match load_certs(&Path::new(&config.general.tls_certificate.unwrap())) { - Ok(certs) => certs, - Err(_) => return Err(Error::TlsError), - }; - - let mut keys = match load_keys(&Path::new(&config.general.tls_private_key.unwrap())) { - Ok(keys) => keys, - Err(_) => return Err(Error::TlsError), - }; - - let config = match rustls::ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_single_cert(certs, keys.remove(0)) - .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err)) { - Ok(c) => c, - Err(_) => return Err(Error::TlsError) + pub fn new() -> Result { + let config = get_config(); + + let certs = match load_certs(&Path::new(&config.general.tls_certificate.unwrap())) { + Ok(certs) => certs, + Err(_) => return Err(Error::TlsError), + }; + + let mut keys = match load_keys(&Path::new(&config.general.tls_private_key.unwrap())) { + Ok(keys) => keys, + Err(_) => return Err(Error::TlsError), + }; + + let config = match rustls::ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(certs, keys.remove(0)) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err)) + { + Ok(c) => c, + Err(_) => return Err(Error::TlsError), }; Ok(Tls { - acceptor: TlsAcceptor::from(Arc::new(config)), + acceptor: TlsAcceptor::from(Arc::new(config)), }) - } + } } struct Stream { - read: Option>, - write: Option, - tls_read: Option>>>, - tls_write: Option>>, + read: Option>, + write: Option, + tls_read: Option>>>, + tls_write: Option>>, } - impl Stream { - pub async fn new(stream: TcpStream, tls: Option) -> Result { - - let config = get_config(); - - match tls { - None => { - let (read, write) = stream.into_split(); - let read = BufReader::new(read); - Ok( - Self { - read: Some(read), - write: Some(write), - tls_read: None, - tls_write: None, - } - ) - } - - Some(tls) => { - let mut tls_stream = match tls.acceptor.accept(stream).await { - Ok(stream) => stream, - Err(_) => return Err(Error::TlsError), - }; - - let (read, write) = split(tls_stream); - - Ok(Self{ - read: None, - write: None, - tls_read: Some(BufReader::new(read)), - tls_write: Some(write), - }) - } - } - } - - async fn read(stream: &mut S) -> Result - where S: tokio::io::AsyncRead + std::marker::Unpin { - - let code = match stream.read_u8().await { - Ok(code) => code, - Err(_) => return Err(Error::SocketError), - }; - - let len = match stream.read_i32().await { - Ok(len) => len, - Err(_) => return Err(Error::SocketError), - }; - - let mut buf = vec![0u8; len as usize - 4]; - - match stream.read_exact(&mut buf).await { - Ok(_) => (), - Err(_) => return Err(Error::SocketError), - }; - - let mut bytes = BytesMut::with_capacity(len as usize + 1); - - bytes.put_u8(code); - bytes.put_i32(len); - bytes.put_slice(&buf); - - Ok(bytes) - } - - async fn write(stream: &mut S, buf: &BytesMut) -> Result<(), Error> - where S: tokio::io::AsyncWrite + std::marker::Unpin { - match stream.write_all(buf).await { - Ok(_) => Ok(()), - Err(_) => return Err(Error::SocketError), - } - } - - pub async fn read_message(&mut self) -> Result { - match &self.read { - Some(read) => Self::read(self.read.as_mut().unwrap()).await, - None => Self::read(self.tls_read.as_mut().unwrap()).await, - } - } - - pub async fn write_all(&mut self, buf: &BytesMut) -> Result<(), Error> { - match &self.write { - Some(write) => Self::write(self.write.as_mut().unwrap(), buf).await, - None => Self::write(self.tls_write.as_mut().unwrap(), buf).await, - } - } -} \ No newline at end of file + pub async fn new(stream: TcpStream, tls: Option) -> Result { + let config = get_config(); + + match tls { + None => { + let (read, write) = stream.into_split(); + let read = BufReader::new(read); + Ok(Self { + read: Some(read), + write: Some(write), + tls_read: None, + tls_write: None, + }) + } + + Some(tls) => { + let mut tls_stream = match tls.acceptor.accept(stream).await { + Ok(stream) => stream, + Err(_) => return Err(Error::TlsError), + }; + + let (read, write) = split(tls_stream); + + Ok(Self { + read: None, + write: None, + tls_read: Some(BufReader::new(read)), + tls_write: Some(write), + }) + } + } + } +} + +// impl tokio::io::AsyncRead for Stream { +// fn poll_read( +// mut self: core::pin::Pin<&mut Self>, +// cx: &mut core::task::Context<'_>, +// buf: &mut tokio::io::ReadBuf<'_> +// ) -> core::task::Poll> { +// match &mut self.get_mut().tls_read { +// None => core::pin::Pin::new(self.read.as_mut().unwrap()).poll_read(cx, buf), +// Some(mut tls) => core::pin::Pin::new(&mut tls).poll_read(cx, buf), +// } +// } +// } From 3cd8fb5a319be73e8336548040eb5612e421559e Mon Sep 17 00:00:00 2001 From: Lev Date: Mon, 27 Jun 2022 16:45:41 -0700 Subject: [PATCH 148/551] hmm --- pgcat.toml | 4 +- src/client.rs | 338 ++++++++++++++++++++++-------------------------- src/main.rs | 57 ++------ src/messages.rs | 7 +- src/stream.rs | 119 ----------------- src/tls.rs | 57 ++++++++ 6 files changed, 229 insertions(+), 353 deletions(-) create mode 100644 src/tls.rs diff --git a/pgcat.toml b/pgcat.toml index b2a6b7662..e9dbf0754 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -33,8 +33,8 @@ ban_time = 60 # Seconds autoreload = false # TLS -tls_certificate = "server.cert" -tls_private_key = "server.key" +# tls_certificate = "server.cert" +# tls_private_key = "server.key" # # User to use for authentication against the server. diff --git a/src/client.rs b/src/client.rs index 4e8ba736a..358982f0b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,12 +1,9 @@ /// Handle clients by pretending to be a PostgreSQL server. use bytes::{Buf, BufMut, BytesMut}; -use log::{debug, error, trace}; +use log::{debug, error, trace, info}; use std::collections::HashMap; -use tokio::io::{split, AsyncReadExt, AsyncWriteExt, BufReader, ReadHalf, WriteHalf}; -use tokio::net::{ - tcp::{OwnedReadHalf, OwnedWriteHalf}, - TcpStream, -}; +use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; +use tokio::net::TcpStream; use crate::admin::handle_admin; use crate::config::get_config; @@ -17,7 +14,7 @@ use crate::pool::{get_pool, ClientServerMap}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; use crate::stats::{get_reporter, Reporter}; -use crate::stream::Tls; +use crate::tls::Tls; use tokio_rustls::server::TlsStream; @@ -41,6 +38,9 @@ pub struct Client { /// them to the backend. buffer: BytesMut, + /// Address + addr: std::net::SocketAddr, + /// The client was started with the sole reason to cancel another running query. cancel_mode: bool, @@ -73,35 +73,107 @@ pub struct Client { last_server_id: Option, } -/// Main client loop. -pub async fn client_loop( +/// Client entrypoint. +pub async fn client_entrypoint( mut stream: TcpStream, client_server_map: ClientServerMap, ) -> Result<(), Error> { + // Figure out if the client wants TLS or not. + let addr = stream.peer_addr().unwrap(); + match get_startup::(&mut stream).await { - Ok((ClientConnectionType::Tls, bytes)) => { - match startup_tls(stream, client_server_map).await { - Ok(mut client) => client.handle().await, - Err(err) => Err(err), + + // Client requested a TLS connection. + Ok((ClientConnectionType::Tls, _)) => { + let config = get_config(); + + // TLS settings are configured, will setup TLS now. + if config.general.tls_certificate != None { + debug!("Accepting TLS request"); + + let mut yes = BytesMut::new(); + yes.put_u8(b'S'); + write_all(&mut stream, yes).await?; + + // Negotiate TLS. + match startup_tls(stream, client_server_map).await { + Ok(mut client) => { + info!("Client {:?} connected (TLS)", addr); + + client.handle().await + } + Err(err) => Err(err), + } + } + + // TLS is not configured, we cannot offer it. + else { + // Rejecting client request for TLS. + let mut no = BytesMut::new(); + no.put_u8(b'N'); + write_all(&mut stream, no).await?; + + // Attempting regular startup. Client can disconnect now + // if they choose. + match get_startup::(&mut stream).await { + // Client accepted unencrypted connection. + Ok((ClientConnectionType::Startup, bytes)) => { + let (read, write) = split(stream); + + // Continue with regular startup. + match Client::startup(read, write, addr, bytes, client_server_map).await { + Ok(mut client) => { + info!("Client {:?} connected (plain)", addr); + + client.handle().await + } + Err(err) => Err(err), + } + } + + // Client probably disconnected rejecting our plain text connection. + _ => Err(Error::ProtocolSyncError), + } } } + // Client wants to use plain connection without encryption. Ok((ClientConnectionType::Startup, bytes)) => { let (read, write) = split(stream); - match Client::handle_startup(read, write, bytes, client_server_map).await { - Ok(mut client) => client.handle().await, + + // Continue with regular startup. + match Client::startup(read, write, addr, bytes, client_server_map).await { + Ok(mut client) => { + info!("Client {:?} connected (plain)", addr); + + client.handle().await + } Err(err) => Err(err), } } + // Client wants to cancel a query. Ok((ClientConnectionType::CancelQuery, bytes)) => { - return Err(Error::ProtocolSyncError); + let (read, write) = split(stream); + + // Continue with cancel query request. + match Client::cancel(read, write, addr, bytes, client_server_map).await { + Ok(mut client) => { + info!("Client {:?} issued a cancel query request", addr); + + client.handle().await + } + + Err(err) => Err(err), + } } + // Something failed, probably the socket. Err(err) => Err(err), } } +/// Handle the first message the client sends. async fn get_startup(stream: &mut S) -> Result<(ClientConnectionType, BytesMut), Error> where S: tokio::io::AsyncRead + std::marker::Unpin + tokio::io::AsyncWrite, @@ -131,32 +203,45 @@ where // Client is requesting to cancel a running query (plain text connection). CANCEL_REQUEST_CODE => Ok((ClientConnectionType::CancelQuery, bytes)), + + // Something else, probably something is wrong and it's not our fault, + // e.g. badly implemented Postgres client. _ => Err(Error::ProtocolSyncError), } } /// Handle TLS connection negotation. pub async fn startup_tls( - mut stream: TcpStream, + stream: TcpStream, client_server_map: ClientServerMap, ) -> Result>, WriteHalf>>, Error> { - // Accept SSL request if SSL is configured. - let mut yes = BytesMut::new(); - yes.put_u8(b'S'); - write_all(&mut stream, yes).await?; - // Negotiate TLS. - let mut tls = Tls::new().unwrap(); + let tls = Tls::new()?; + let addr = stream.peer_addr().unwrap(); + let mut stream = match tls.acceptor.accept(stream).await { Ok(stream) => stream, - Err(_) => return Err(Error::TlsError), + + // TLS negotitation failed. + Err(err) => { + error!("TLS negotiation failed: {:?}", err); + return Err(Error::TlsError) + } }; + // TLS negotitation successful. + // Continue with regular startup using encrypted connection. match get_startup::>(&mut stream).await { + + // Got good startup message, proceeding like normal except we + // are encrypted now. Ok((ClientConnectionType::Startup, bytes)) => { let (read, write) = split(stream); - Client::handle_startup(read, write, bytes, client_server_map).await + + Client::startup(read, write, addr, bytes, client_server_map).await } + + // Bad Postgres client. _ => Err(Error::ProtocolSyncError), } } @@ -166,10 +251,12 @@ where S: tokio::io::AsyncRead + std::marker::Unpin, T: tokio::io::AsyncWrite + std::marker::Unpin, { - // Perform client startup sequence in TLS. - pub async fn handle_startup( + /// Handle Postgres client startup after TLS negotiation is complete + /// or over plain text. + pub async fn startup( mut read: S, mut write: T, + addr: std::net::SocketAddr, bytes: BytesMut, // The rest of the startup message. client_server_map: ClientServerMap, ) -> Result, Error> { @@ -244,6 +331,7 @@ where return Ok(Client { read: BufReader::new(read), write: write, + addr, buffer: BytesMut::with_capacity(8196), cancel_mode: false, transaction_mode: transaction_mode, @@ -258,161 +346,38 @@ where }); } - /// Perform client startup sequence. - /// See docs: - // pub async fn startup( - // mut stream: TcpStream, - // client_server_map: ClientServerMap, - // ) -> Result, WriteHalf>, Error> { - // let config = get_config(); - // let transaction_mode = config.general.pool_mode == "transaction"; - // let stats = get_reporter(); - - // loop { - // trace!("Waiting for StartupMessage"); - - // // Could be StartupMessage, SSLRequest or CancelRequest. - // let len = match stream.read_i32().await { - // Ok(len) => len, - // Err(_) => return Err(Error::ClientBadStartup), - // }; - - // let mut startup = vec![0u8; len as usize - 4]; - - // match stream.read_exact(&mut startup).await { - // Ok(_) => (), - // Err(_) => return Err(Error::ClientBadStartup), - // }; - - // let mut bytes = BytesMut::from(&startup[..]); - // let code = bytes.get_i32(); - - // match code { - // // Client wants SSL. We don't support it at the moment. - // SSL_REQUEST_CODE => { - // trace!("Rejecting SSLRequest"); - - // let mut no = BytesMut::with_capacity(1); - // no.put_u8(b'N'); - - // write_all(&mut stream, no).await?; - // } - - // // Regular startup message. - // PROTOCOL_VERSION_NUMBER => { - // trace!("Got StartupMessage"); - // let parameters = parse_startup(bytes.clone())?; - - // // Generate random backend ID and secret key - // let process_id: i32 = rand::random(); - // let secret_key: i32 = rand::random(); - - // // Perform MD5 authentication. - // // TODO: Add SASL support. - // let salt = md5_challenge(&mut stream).await?; - - // let code = match stream.read_u8().await { - // Ok(p) => p, - // Err(_) => return Err(Error::SocketError), - // }; - - // // PasswordMessage - // if code as char != 'p' { - // debug!("Expected p, got {}", code as char); - // return Err(Error::ProtocolSyncError); - // } - - // let len = match stream.read_i32().await { - // Ok(len) => len, - // Err(_) => return Err(Error::SocketError), - // }; - - // let mut password_response = vec![0u8; (len - 4) as usize]; - - // match stream.read_exact(&mut password_response).await { - // Ok(_) => (), - // Err(_) => return Err(Error::SocketError), - // }; - - // // Compare server and client hashes. - // let password_hash = - // md5_hash_password(&config.user.name, &config.user.password, &salt); - - // if password_hash != password_response { - // debug!("Password authentication failed"); - // wrong_password(&mut stream, &config.user.name).await?; - // return Err(Error::ClientError); - // } - - // debug!("Password authentication successful"); - - // auth_ok(&mut stream).await?; - // write_all(&mut stream, get_pool().server_info()).await?; - // backend_key_data(&mut stream, process_id, secret_key).await?; - // ready_for_query(&mut stream).await?; - - // trace!("Startup OK"); - - // let database = parameters - // .get("database") - // .unwrap_or(parameters.get("user").unwrap()); - // let admin = ["pgcat", "pgbouncer"] - // .iter() - // .filter(|db| *db == &database) - // .count() - // == 1; - - // // Split the read and write streams - // // so we can control buffering. - // let (read, write) = split(stream); - - // return Ok(Client { - // read: BufReader::new(read), - // write: write, - // buffer: BytesMut::with_capacity(8196), - // cancel_mode: false, - // transaction_mode: transaction_mode, - // process_id: process_id, - // secret_key: secret_key, - // client_server_map: client_server_map, - // parameters: parameters, - // stats: stats, - // admin: admin, - // last_address_id: None, - // last_server_id: None, - // }); - // } - - // // Query cancel request. - // CANCEL_REQUEST_CODE => { - // let (read, write) = split(stream); - - // let process_id = bytes.get_i32(); - // let secret_key = bytes.get_i32(); - - // return Ok(Client { - // read: BufReader::new(read), - // write: write, - // buffer: BytesMut::with_capacity(8196), - // cancel_mode: true, - // transaction_mode: transaction_mode, - // process_id: process_id, - // secret_key: secret_key, - // client_server_map: client_server_map, - // parameters: HashMap::new(), - // stats: stats, - // admin: false, - // last_address_id: None, - // last_server_id: None, - // }); - // } - - // _ => { - // return Err(Error::ProtocolSyncError); - // } - // }; - // } - // } + /// Handle cancel request. + pub async fn cancel( + read: S, + write: T, + addr: std::net::SocketAddr, + mut bytes: BytesMut, // The rest of the startup message. + client_server_map: ClientServerMap, + ) -> Result, Error> { + let process_id = bytes.get_i32(); + let secret_key = bytes.get_i32(); + + let config = get_config(); + let transaction_mode = config.general.pool_mode == "transaction"; + let stats = get_reporter(); + + return Ok(Client { + read: BufReader::new(read), + write: write, + addr, + buffer: BytesMut::with_capacity(8196), + cancel_mode: true, + transaction_mode: transaction_mode, + process_id: process_id, + secret_key: secret_key, + client_server_map: client_server_map, + parameters: HashMap::new(), + stats: stats, + admin: false, + last_address_id: None, + last_server_id: None, + }); + } /// Handle a connected and authenticated client. pub async fn handle(&mut self) -> Result<(), Error> { @@ -608,8 +573,8 @@ where self.last_server_id = Some(server.process_id()); debug!( - "Client stuff talking to server {:?}", - // self.write.peer_addr().unwrap(), + "Client {:?} talking to server {:?}", + self.addr, server.address() ); @@ -846,6 +811,9 @@ where impl Drop for Client { fn drop(&mut self) { + let mut guard = self.client_server_map.lock(); + guard.remove(&(self.process_id, self.secret_key)); + // Update statistics. if let Some(address_id) = self.last_address_id { self.stats.client_disconnecting(self.process_id, address_id); @@ -854,5 +822,7 @@ impl Drop for Client { self.stats.server_idle(process_id, address_id); } } + + // self.release(); } } diff --git a/src/main.rs b/src/main.rs index 439c01ac3..7fb0ec6e9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,11 +45,6 @@ use tokio::{ sync::mpsc, }; -use tokio::net::{ - tcp::{OwnedReadHalf, OwnedWriteHalf}, - TcpStream, -}; - use std::collections::HashMap; use std::sync::Arc; @@ -65,9 +60,8 @@ mod scram; mod server; mod sharding; mod stats; -mod stream; +mod tls; -use crate::constants::*; use config::{get_config, reload_config}; use pool::{ClientServerMap, ConnectionPool}; use stats::{Collector, Reporter, REPORTER}; @@ -159,45 +153,22 @@ async fn main() { // Handle client. tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); - // match client::get_startup(&mut socket) { - // Ok((code, bytes)) => match code { - // SSL_REQUEST_CODE => client::Client::tls_startup< - // } - // } - - match client::client_loop(socket, client_server_map).await { - Ok(_) => (), + + match client::client_entrypoint(socket, client_server_map).await { + Ok(_) => { + let duration = chrono::offset::Utc::now().naive_utc() - start; + + info!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + }, + Err(err) => { - debug!("Client failed to login: {:?}", err); + debug!("Client disconnected with error {:?}", err); } }; - - // match client::Client::::startup(socket, client_server_map).await { - // Ok(mut client) => { - // info!("Client {:?} connected", addr); - - // match client.handle().await { - // Ok(()) => { - // let duration = chrono::offset::Utc::now().naive_utc() - start; - - // info!( - // "Client {:?} disconnected, session duration: {}", - // addr, - // format_duration(&duration) - // ); - // } - - // Err(err) => { - // error!("Client disconnected with error: {:?}", err); - // client.release(); - // } - // } - // } - - // Err(err) => { - // debug!("Client failed to login: {:?}", err); - // } - // }; }); } }); diff --git a/src/messages.rs b/src/messages.rs index 7b0479299..89795c65e 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -2,11 +2,8 @@ /// and handle TcpStream (TCP socket). use bytes::{Buf, BufMut, BytesMut}; use md5::{Digest, Md5}; -use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; -use tokio::net::{ - tcp::{OwnedReadHalf, OwnedWriteHalf}, - TcpStream, -}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; use crate::errors::Error; use std::collections::HashMap; diff --git a/src/stream.rs b/src/stream.rs index 9c19b89ed..e69de29bb 100644 --- a/src/stream.rs +++ b/src/stream.rs @@ -1,119 +0,0 @@ -// Stream wrapper. - -use bytes::{Buf, BufMut, BytesMut}; -use rustls_pemfile::{certs, rsa_private_keys}; -use std::path::Path; -use std::sync::Arc; -use tokio::io::{split, AsyncReadExt, AsyncWriteExt, BufReader, ReadHalf, WriteHalf}; -use tokio::net::{ - tcp::{OwnedReadHalf, OwnedWriteHalf}, - TcpStream, -}; -use tokio_rustls::rustls::{self, Certificate, PrivateKey}; -use tokio_rustls::server::TlsStream; -use tokio_rustls::TlsAcceptor; - -use crate::config::get_config; -use crate::errors::Error; - -// TLS -fn load_certs(path: &std::path::Path) -> std::io::Result> { - certs(&mut std::io::BufReader::new(std::fs::File::open(path)?)) - .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid cert")) - .map(|mut certs| certs.drain(..).map(Certificate).collect()) -} - -fn load_keys(path: &std::path::Path) -> std::io::Result> { - rsa_private_keys(&mut std::io::BufReader::new(std::fs::File::open(path)?)) - .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid key")) - .map(|mut keys| keys.drain(..).map(PrivateKey).collect()) -} - -pub struct Tls { - pub acceptor: TlsAcceptor, -} - -impl Tls { - pub fn new() -> Result { - let config = get_config(); - - let certs = match load_certs(&Path::new(&config.general.tls_certificate.unwrap())) { - Ok(certs) => certs, - Err(_) => return Err(Error::TlsError), - }; - - let mut keys = match load_keys(&Path::new(&config.general.tls_private_key.unwrap())) { - Ok(keys) => keys, - Err(_) => return Err(Error::TlsError), - }; - - let config = match rustls::ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth() - .with_single_cert(certs, keys.remove(0)) - .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err)) - { - Ok(c) => c, - Err(_) => return Err(Error::TlsError), - }; - - Ok(Tls { - acceptor: TlsAcceptor::from(Arc::new(config)), - }) - } -} - -struct Stream { - read: Option>, - write: Option, - tls_read: Option>>>, - tls_write: Option>>, -} - -impl Stream { - pub async fn new(stream: TcpStream, tls: Option) -> Result { - let config = get_config(); - - match tls { - None => { - let (read, write) = stream.into_split(); - let read = BufReader::new(read); - Ok(Self { - read: Some(read), - write: Some(write), - tls_read: None, - tls_write: None, - }) - } - - Some(tls) => { - let mut tls_stream = match tls.acceptor.accept(stream).await { - Ok(stream) => stream, - Err(_) => return Err(Error::TlsError), - }; - - let (read, write) = split(tls_stream); - - Ok(Self { - read: None, - write: None, - tls_read: Some(BufReader::new(read)), - tls_write: Some(write), - }) - } - } - } -} - -// impl tokio::io::AsyncRead for Stream { -// fn poll_read( -// mut self: core::pin::Pin<&mut Self>, -// cx: &mut core::task::Context<'_>, -// buf: &mut tokio::io::ReadBuf<'_> -// ) -> core::task::Poll> { -// match &mut self.get_mut().tls_read { -// None => core::pin::Pin::new(self.read.as_mut().unwrap()).poll_read(cx, buf), -// Some(mut tls) => core::pin::Pin::new(&mut tls).poll_read(cx, buf), -// } -// } -// } diff --git a/src/tls.rs b/src/tls.rs new file mode 100644 index 000000000..ab9fc4001 --- /dev/null +++ b/src/tls.rs @@ -0,0 +1,57 @@ +// Stream wrapper. + +use rustls_pemfile::{certs, rsa_private_keys}; +use std::path::Path; +use std::sync::Arc; +use tokio_rustls::rustls::{self, Certificate, PrivateKey}; +use tokio_rustls::TlsAcceptor; + +use crate::config::get_config; +use crate::errors::Error; + +// TLS +fn load_certs(path: &Path) -> std::io::Result> { + certs(&mut std::io::BufReader::new(std::fs::File::open(path)?)) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid cert")) + .map(|mut certs| certs.drain(..).map(Certificate).collect()) +} + +fn load_keys(path: &Path) -> std::io::Result> { + rsa_private_keys(&mut std::io::BufReader::new(std::fs::File::open(path)?)) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid key")) + .map(|mut keys| keys.drain(..).map(PrivateKey).collect()) +} + +pub struct Tls { + pub acceptor: TlsAcceptor, +} + +impl Tls { + pub fn new() -> Result { + let config = get_config(); + + let certs = match load_certs(&Path::new(&config.general.tls_certificate.unwrap())) { + Ok(certs) => certs, + Err(_) => return Err(Error::TlsError), + }; + + let mut keys = match load_keys(&Path::new(&config.general.tls_private_key.unwrap())) { + Ok(keys) => keys, + Err(_) => return Err(Error::TlsError), + }; + + let config = match rustls::ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(certs, keys.remove(0)) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err)) + { + Ok(c) => c, + Err(_) => return Err(Error::TlsError), + }; + + Ok(Tls { + acceptor: TlsAcceptor::from(Arc::new(config)), + }) + } +} From 7337312b94fe116e86996e8678f3458fce784913 Mon Sep 17 00:00:00 2001 From: Lev Date: Mon, 27 Jun 2022 16:46:03 -0700 Subject: [PATCH 149/551] bye --- src/stream.rs | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/stream.rs diff --git a/src/stream.rs b/src/stream.rs deleted file mode 100644 index e69de29bb..000000000 From 908d528dc6ffe14fa55a188b5cb2274f7c1615e6 Mon Sep 17 00:00:00 2001 From: Lev Date: Mon, 27 Jun 2022 17:01:14 -0700 Subject: [PATCH 150/551] config check --- src/config.rs | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/tls.rs | 4 ++-- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 170054184..0e4b8d233 100644 --- a/src/config.rs +++ b/src/config.rs @@ -8,9 +8,11 @@ use std::sync::Arc; use tokio::fs::File; use tokio::io::AsyncReadExt; use toml; +use std::path::Path; use crate::errors::Error; use crate::{ClientServerMap, ConnectionPool}; +use crate::tls::{load_certs, load_keys}; /// Globally available configuration. static CONFIG: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Config::default())); @@ -253,6 +255,25 @@ impl Config { info!("Primary reads: {}", self.query_router.primary_reads_enabled); info!("Query router: {}", self.query_router.query_parser_enabled); info!("Number of shards: {}", self.shards.len()); + + match self.general.tls_certificate.clone() { + Some(tls_certificate) => { + info!("TLS certificate: {}", tls_certificate); + + match self.general.tls_private_key.clone() { + Some(tls_private_key) => { + info!("TLS private key: {}", tls_private_key); + info!("TLS support is enabled"); + }, + + None => (), + } + } + + None => { + info!("TLS support is disabled"); + }, + }; } } @@ -372,6 +393,39 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; + // Validate TLS! + match config.general.tls_certificate.clone() { + Some(tls_certificate) => { + match load_certs(&Path::new(&tls_certificate)) { + Ok(_) => { + // Cert is okay, but what about the private key? + match config.general.tls_private_key.clone() { + Some(tls_private_key) => { + match load_keys(&Path::new(&tls_private_key)) { + Ok(_) => (), + Err(err) => { + error!("tls_private_key is incorrectly configured: {:?}", err); + return Err(Error::BadConfig); + } + } + } + + None => { + error!("tls_certificate is set, but the tls_private_key is not"); + return Err(Error::BadConfig); + } + }; + } + + Err(err) => { + error!("tls_certificate is incorrectly configured: {:?}", err); + return Err(Error::BadConfig); + } + } + }, + None => (), + }; + config.path = path.to_string(); // Update the configuration globally. diff --git a/src/tls.rs b/src/tls.rs index ab9fc4001..3bc4a6a0f 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -10,13 +10,13 @@ use crate::config::get_config; use crate::errors::Error; // TLS -fn load_certs(path: &Path) -> std::io::Result> { +pub fn load_certs(path: &Path) -> std::io::Result> { certs(&mut std::io::BufReader::new(std::fs::File::open(path)?)) .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid cert")) .map(|mut certs| certs.drain(..).map(Certificate).collect()) } -fn load_keys(path: &Path) -> std::io::Result> { +pub fn load_keys(path: &Path) -> std::io::Result> { rsa_private_keys(&mut std::io::BufReader::new(std::fs::File::open(path)?)) .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid key")) .map(|mut keys| keys.drain(..).map(PrivateKey).collect()) From ddded95c974c495425157f2aace3ae4db9a3a118 Mon Sep 17 00:00:00 2001 From: Lev Date: Mon, 27 Jun 2022 17:01:40 -0700 Subject: [PATCH 151/551] lint --- src/client.rs | 7 ++----- src/config.rs | 24 +++++++++++------------- src/main.rs | 2 +- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/client.rs b/src/client.rs index 358982f0b..05895b6d6 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,6 +1,6 @@ /// Handle clients by pretending to be a PostgreSQL server. use bytes::{Buf, BufMut, BytesMut}; -use log::{debug, error, trace, info}; +use log::{debug, error, info, trace}; use std::collections::HashMap; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; use tokio::net::TcpStream; @@ -82,7 +82,6 @@ pub async fn client_entrypoint( let addr = stream.peer_addr().unwrap(); match get_startup::(&mut stream).await { - // Client requested a TLS connection. Ok((ClientConnectionType::Tls, _)) => { let config = get_config(); @@ -105,7 +104,6 @@ pub async fn client_entrypoint( Err(err) => Err(err), } } - // TLS is not configured, we cannot offer it. else { // Rejecting client request for TLS. @@ -225,14 +223,13 @@ pub async fn startup_tls( // TLS negotitation failed. Err(err) => { error!("TLS negotiation failed: {:?}", err); - return Err(Error::TlsError) + return Err(Error::TlsError); } }; // TLS negotitation successful. // Continue with regular startup using encrypted connection. match get_startup::>(&mut stream).await { - // Got good startup message, proceeding like normal except we // are encrypted now. Ok((ClientConnectionType::Startup, bytes)) => { diff --git a/src/config.rs b/src/config.rs index 0e4b8d233..da59d2ae1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,15 +4,15 @@ use log::{error, info}; use once_cell::sync::Lazy; use serde_derive::Deserialize; use std::collections::{HashMap, HashSet}; +use std::path::Path; use std::sync::Arc; use tokio::fs::File; use tokio::io::AsyncReadExt; use toml; -use std::path::Path; use crate::errors::Error; -use crate::{ClientServerMap, ConnectionPool}; use crate::tls::{load_certs, load_keys}; +use crate::{ClientServerMap, ConnectionPool}; /// Globally available configuration. static CONFIG: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Config::default())); @@ -264,7 +264,7 @@ impl Config { Some(tls_private_key) => { info!("TLS private key: {}", tls_private_key); info!("TLS support is enabled"); - }, + } None => (), } @@ -272,7 +272,7 @@ impl Config { None => { info!("TLS support is disabled"); - }, + } }; } } @@ -400,15 +400,13 @@ pub async fn parse(path: &str) -> Result<(), Error> { Ok(_) => { // Cert is okay, but what about the private key? match config.general.tls_private_key.clone() { - Some(tls_private_key) => { - match load_keys(&Path::new(&tls_private_key)) { - Ok(_) => (), - Err(err) => { - error!("tls_private_key is incorrectly configured: {:?}", err); - return Err(Error::BadConfig); - } + Some(tls_private_key) => match load_keys(&Path::new(&tls_private_key)) { + Ok(_) => (), + Err(err) => { + error!("tls_private_key is incorrectly configured: {:?}", err); + return Err(Error::BadConfig); } - } + }, None => { error!("tls_certificate is set, but the tls_private_key is not"); @@ -422,7 +420,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { return Err(Error::BadConfig); } } - }, + } None => (), }; diff --git a/src/main.rs b/src/main.rs index 7fb0ec6e9..49795b70d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -163,7 +163,7 @@ async fn main() { addr, format_duration(&duration) ); - }, + } Err(err) => { debug!("Client disconnected with error {:?}", err); From d8dc629362cf60382ef9e6a547450c487d1e2eab Mon Sep 17 00:00:00 2001 From: Lev Date: Mon, 27 Jun 2022 17:05:45 -0700 Subject: [PATCH 152/551] lock em up --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b193e778c..3f65e90c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,5 +29,5 @@ hmac = "0.12" sha2 = "0.10" base64 = "0.13" stringprep = "0.1" -tokio-rustls = "*" -rustls-pemfile = "*" +tokio-rustls = "0.23" +rustls-pemfile = "1" From c63e1df8fcdce14498d91e0393281473389b44b4 Mon Sep 17 00:00:00 2001 From: Lev Date: Mon, 27 Jun 2022 17:06:49 -0700 Subject: [PATCH 153/551] Im about to get a nasty email --- .circleci/pgcat.toml | 3 +++ .circleci/server.cert | 21 +++++++++++++++++++++ .circleci/server.key | 28 ++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 .circleci/server.cert create mode 100644 .circleci/server.key diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index d7249f152..943eff922 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -32,6 +32,9 @@ ban_time = 60 # Seconds # autoreload = true +tls_certificate = ".circleci/server.cert" +tls_private_key = ".circleci/server.key" + # # User to use for authentication against the server. [user] diff --git a/.circleci/server.cert b/.circleci/server.cert new file mode 100644 index 000000000..a24847a75 --- /dev/null +++ b/.circleci/server.cert @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUChIvUGFJGJe5EDch32rchqoxER0wDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjA2MjcyMjI2MDZaFw0yMjA3 +MjcyMjI2MDZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDdTwrBzV1v79faVckFvIn/9V4fypYs4vDi3X+h3wGn +AjEh6mmizlKCwSwAam07D9Q5zKiXFrzNJqzSioOv5zsOAvObwrnzbtKSwfs3aP5g +eEh2clHCZYx9p06WszPcgSB5nTz1NeY4XAwvGn3A+SVCLyPMTNwnem48+ONh2F9u +FHtSuIsEVvTjMlH09O7LjwJlODxy3HNv2JHYM5Hx9tzc+NVYdERPtaVcX8ycw1Eh +9hgGSgfaNM52/JfRMIDhENrsn0S1omRUtcJe72loreiwrECUOLAnAfp9Xqc+rMPP +aLA6ElzmYef1+ZEC0p6isCHPhxY5ESVhKYhE9nQvksjnAgMBAAGjUzBRMB0GA1Ud +DgQWBBQLDtzexqjx7xPtUZuZB/angU9oSDAfBgNVHSMEGDAWgBQLDtzexqjx7xPt +UZuZB/angU9oSDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC/ +mxY/a/WeLENVj2Gg9EUH0CKzfqeTey1mb6YfPGxzrD7oq1m0Vn2MmTbjZrJgh/Ob +QckO3ElF4kC9+6XP+iDPmabGpjeLgllBboT5l2aqnD1syMrf61WPLzgRzRfplYGy +cjBQDDKPu8Lu0QRMWU28tHYN0bMxJoCuXysGGX5WsuFnKCA6f/V+nycJJXxJH3eB +eLjTueD9/RE3OXhi6m8A29Q1E9AE5EF4uRxYXrr91BmYnk4aFvSmBxhUEzE12eSN +lHB/uSc0+Dp+UVmVr6wW8AQfd16UBA0BUf3kSW3aSvirYPYH0rXiOOpEJgOwOMnR +f5+XAbN1Y+3OsFz/ZmP9 +-----END CERTIFICATE----- diff --git a/.circleci/server.key b/.circleci/server.key new file mode 100644 index 000000000..14e4fd688 --- /dev/null +++ b/.circleci/server.key @@ -0,0 +1,28 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDdTwrBzV1v79fa +VckFvIn/9V4fypYs4vDi3X+h3wGnAjEh6mmizlKCwSwAam07D9Q5zKiXFrzNJqzS +ioOv5zsOAvObwrnzbtKSwfs3aP5geEh2clHCZYx9p06WszPcgSB5nTz1NeY4XAwv +Gn3A+SVCLyPMTNwnem48+ONh2F9uFHtSuIsEVvTjMlH09O7LjwJlODxy3HNv2JHY +M5Hx9tzc+NVYdERPtaVcX8ycw1Eh9hgGSgfaNM52/JfRMIDhENrsn0S1omRUtcJe +72loreiwrECUOLAnAfp9Xqc+rMPPaLA6ElzmYef1+ZEC0p6isCHPhxY5ESVhKYhE +9nQvksjnAgMBAAECggEAbnvddO9frFhivJ+DIhgEFQKcIOb0nigV9kx6QYehvYy8 +lp/+aMb0Lk7d9r8rFQdL/icMK5GwZALg2KNKJvEbbF1Q3PwT9VHoUlgBYKJMDEFA +e9GKu7ASuVBjTZzdUUItwkkbe5eS/aQGeSWSjlpTnX0HNCFS72qRymK+scRhsAQf +ZoHyZHDslkvPR3Pos+sndWBYCDHag5/KoPhsMt1+5S9NQcOUHx9Ac0gLHjau3N+P +0FhODHFFGnnpyQvLvj6u3ZOR34ladMgoBglE0O3vPFhckn92EK4teeTWOsUMotiz +qM3QIJTOJjtiY6VDGY93bIa4pFvt7Zi4vIerenKt0QKBgQD/UMFqfevTAMrk10AC +bOa4+cM07ORY4ZwVj5ILhZn+8crDEEtBsUyuEU2FTINtnoEq1yGc/IXpsyS1BHjL +L1xSml5LN3jInbi8z5XQfY5Sj3VOMtwY6yD20jcdeDC44rz3nStXdkcMWxbTMapx +iOPsap5ciUKOMS7LyMidPEG/LQKBgQDd5vHgrLN0FBIIm+vZg6MEm4QyobstVp4l +7V/GZsdL+M8AQv1Rx+5wSUSWKomOIv5lglis7f6g0c9O7Qkr78/wzoyoKC2RRqPp +I90GjY2Iv22N4GIkRrDAgMZbkTitzIB6tbXEVeLAOh3frFJ8IwauRCOiXIjrZdJ4 +FvV86+nU4wKBgQDdWTP2kWkMrBk7QOp7r9Jv+AmnLuHhtOdPQgOJ/bA++X2ik9PL +Bl3GY7XjpSwks1CkxZKcucmXjPp7/X6EGXFfI/owF82dkDADca0e7lufdERtIWb0 +K5WOpz2lTPhgsiLGQfq7fw2lxqsJOnvcpqOD6gOVkmKjSDyb7F0RBJazmQKBgQDD +a8PQTcesjpBjLI3EfX1vbVY7ENu6zfFxDV+vZoxVh8UlQdm90AlYse3JIaUKnB7W +Xrihcucv0hZ0N6RAIW5LcFvHK7sVmdR4WbEpODhRGeTtcZJ8yBSZM898jKQRy2vK +pYRyaADNsWDlvujVkjMr/a40KrIaPQ3h3LZNUaYYaQKBgQD1x8A5S5SiE1cN1vFr +aACkmA2WqEDKKhUsUigJdwW6WB/B9kWlIlz/iV1H9uwBXtSIYG4VqCSTAvh0z4gX +Qu2SrdPm5PYnKzpdynpz78OnGdflD1RKWFGHItR6GN6tj/VmulO6mlFvT4jzBQ7j ++Hf8m2TcD4U3ksz3xw+YOD+cmA== +-----END RSA PRIVATE KEY----- From 2e2a7cd16e6c1d6ea50f4877012a844c180c66fe Mon Sep 17 00:00:00 2001 From: Lev Date: Mon, 27 Jun 2022 17:07:40 -0700 Subject: [PATCH 154/551] forgotten comment --- src/admin.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/admin.rs b/src/admin.rs index 622c14a97..74acf1512 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -2,7 +2,6 @@ use bytes::{Buf, BufMut, BytesMut}; use log::{info, trace}; use std::collections::HashMap; -// use tokio::net::tcp::T; use crate::config::{get_config, reload_config}; use crate::errors::Error; From 9ae24d8ede9975d437f8ae27151aa1393c9d6d82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Jul 2022 13:24:41 -0700 Subject: [PATCH 155/551] Bump activerecord from 7.0.2.2 to 7.0.3.1 in /tests/ruby (#94) Bumps [activerecord](https://github.com/rails/rails) from 7.0.2.2 to 7.0.3.1. - [Release notes](https://github.com/rails/rails/releases) - [Changelog](https://github.com/rails/rails/blob/v7.0.3.1/activerecord/CHANGELOG.md) - [Commits](https://github.com/rails/rails/compare/v7.0.2.2...v7.0.3.1) --- updated-dependencies: - dependency-name: activerecord dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- tests/ruby/Gemfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock index 045e2f61a..607df18c2 100644 --- a/tests/ruby/Gemfile.lock +++ b/tests/ruby/Gemfile.lock @@ -1,21 +1,21 @@ GEM remote: https://rubygems.org/ specs: - activemodel (7.0.2.2) - activesupport (= 7.0.2.2) - activerecord (7.0.2.2) - activemodel (= 7.0.2.2) - activesupport (= 7.0.2.2) - activesupport (7.0.2.2) + activemodel (7.0.3.1) + activesupport (= 7.0.3.1) + activerecord (7.0.3.1) + activemodel (= 7.0.3.1) + activesupport (= 7.0.3.1) + activesupport (7.0.3.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) ast (2.4.2) - concurrent-ruby (1.1.9) - i18n (1.10.0) + concurrent-ruby (1.1.10) + i18n (1.11.0) concurrent-ruby (~> 1.0) - minitest (5.15.0) + minitest (5.16.2) parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) From af5af7b72bbaefe06f2ca283130706c1ae48b486 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 25 Jul 2022 22:25:59 -0700 Subject: [PATCH 156/551] Update Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 536647478..ec29182a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM rust:1.58-slim-buster AS builder +FROM rust:1 AS builder COPY . /app WORKDIR /app RUN cargo build --release -FROM debian:buster-slim +FROM debian:bullseye-slim COPY --from=builder /app/target/release/pgcat /usr/bin/pgcat COPY --from=builder /app/pgcat.toml /etc/pgcat/pgcat.toml WORKDIR /etc/pgcat From 909b5f0ba364bbb76ee2454daf45960269c610ab Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 27 Jul 2022 21:47:55 -0500 Subject: [PATCH 157/551] Add support for multi-database / multi-user pools (#96) * Add support for multi-database / multi-user pools * Nothing * cargo fmt * CI * remove test users * rename pool * Update tests to use admin user/pass * more fixes * Revert bad change * Use PGDATABASE env var * send server info in case of admin --- .circleci/pgcat.toml | 102 +++++------ .circleci/run_tests.sh | 23 ++- Cargo.lock | 2 +- Cargo.toml | 2 +- pgcat.toml | 126 +++++++------ src/admin.rs | 151 +++++++-------- src/client.rs | 101 ++++++---- src/config.rs | 407 +++++++++++++++++++++++++---------------- src/main.rs | 4 +- src/pool.rs | 241 ++++++++++++++---------- src/query_router.rs | 46 ++--- src/server.rs | 4 +- src/stats.rs | 10 +- tests/ruby/tests.rb | 4 +- 14 files changed, 710 insertions(+), 513 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 943eff922..eca8f6736 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -5,21 +5,12 @@ # # General pooler settings [general] - # What IP to run on, 0.0.0.0 means accessible from everywhere. host = "0.0.0.0" # Port to run on, same as PgBouncer used in this example. port = 6432 -# How many connections to allocate per server. -pool_size = 15 - -# Pool mode (see PgBouncer docs for more). -# session: one server connection per connected client -# transaction: one server connection per client transaction -pool_mode = "transaction" - # How long to wait before aborting a server connection (ms). connect_timeout = 100 @@ -29,56 +20,27 @@ healthcheck_timeout = 100 # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # Seconds -# +# Reload config automatically if it changes. autoreload = true +# TLS tls_certificate = ".circleci/server.cert" tls_private_key = ".circleci/server.key" -# -# User to use for authentication against the server. -[user] -name = "sharding_user" -password = "sharding_user" - - -# -# Shards in the cluster -[shards] - -# Shard 0 -[shards.0] - -# [ host, port, role ] -servers = [ - [ "127.0.0.1", 5432, "primary" ], - [ "localhost", 5433, "replica" ], - # [ "127.0.1.1", 5432, "replica" ], -] -# Database name (e.g. "postgres") -database = "shard0" - -[shards.1] -# [ host, port, role ] -servers = [ - [ "127.0.0.1", 5432, "primary" ], - [ "localhost", 5433, "replica" ], - # [ "127.0.1.1", 5432, "replica" ], -] -database = "shard1" - -[shards.2] -# [ host, port, role ] -servers = [ - [ "127.0.0.1", 5432, "primary" ], - [ "localhost", 5433, "replica" ], - # [ "127.0.1.1", 5432, "replica" ], -] -database = "shard2" - +# Credentials to access the virtual administrative database (pgbouncer or pgcat) +# Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. +admin_username = "admin_user" +admin_password = "admin_pass" -# Settings for our query routing layer. -[query_router] +# pool +# configs are structured as pool. +# the pool_name is what clients use as database name when connecting +# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded" +[pools.sharded_db] +# Pool mode (see PgBouncer docs for more). +# session: one server connection per connected client +# transaction: one server connection per client transaction +pool_mode = "transaction" # If the client doesn't specify, route traffic to # this role by default. @@ -88,7 +50,6 @@ database = "shard2" # primary: all queries go to the primary unless otherwise specified. default_role = "any" - # Query parser. If enabled, we'll attempt to parse # every incoming query to determine if it's a read or a write. # If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, @@ -109,3 +70,36 @@ primary_reads_enabled = true # sha1: A hashing function based on SHA1 # sharding_function = "pg_bigint_hash" + +# Credentials for users that may connect to this cluster +[pools.sharded_db.users.0] +username = "sharding_user" +password = "sharding_user" +# Maximum number of server connections that can be established for this user +# The maximum number of connection from a single Pgcat process to any database in the cluster +# is the sum of pool_size across all users. +pool_size = 9 + +# Shard 0 +[pools.sharded_db.shards.0] +# [ host, port, role ] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ] +] +# Database name (e.g. "postgres") +database = "shard0" + +[pools.sharded_db.shards.1] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ], +] +database = "shard1" + +[pools.sharded_db.shards.2] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ], +] +database = "shard2" diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index a0e23f0ac..c932a86ee 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -32,6 +32,7 @@ toxiproxy-cli create -l 127.0.0.1:5433 -u 127.0.0.1:5432 postgres_replica start_pgcat "info" export PGPASSWORD=sharding_user +export PGDATABASE=sharded_db # pgbench test pgbench -U sharding_user -i -h 127.0.0.1 -p 6432 @@ -47,7 +48,7 @@ sleep 1 killall psql -s SIGINT # Reload pool (closing unused server connections) -psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' +PGPASSWORD=admin_pass psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' (psql -U sharding_user -h 127.0.0.1 -p 6432 -c 'SELECT pg_sleep(50)' || true) & sleep 1 @@ -72,15 +73,17 @@ cd tests/ruby && \ cd ../.. # Admin tests -psql -U sharding_user -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/null -psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null -psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW CONFIG' > /dev/null -psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' > /dev/null -psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW LISTS' > /dev/null -psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW POOLS' > /dev/null -psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW VERSION' > /dev/null -psql -U sharding_user -h 127.0.0.1 -p 6432 -d pgbouncer -c "SET client_encoding TO 'utf8'" > /dev/null # will ignore -(! psql -U sharding_user -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) +export PGPASSWORD=admin_pass +psql -U admin_user -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW CONFIG' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW LISTS' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW POOLS' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW VERSION' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c "SET client_encoding TO 'utf8'" > /dev/null # will ignore +(! psql -U admin_user -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) +export PGPASSWORD=sharding_user # Start PgCat in debug to demonstrate failover better start_pgcat "trace" diff --git a/Cargo.lock b/Cargo.lock index cfaa0fbdc..ddab730d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -395,7 +395,7 @@ dependencies = [ [[package]] name = "pgcat" -version = "0.4.0-beta1" +version = "0.6.0-alpha1" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 3f65e90c9..8bdeab676 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "0.4.0-beta1" +version = "0.6.0-alpha1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/pgcat.toml b/pgcat.toml index e9dbf0754..a1937e6cb 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -5,21 +5,12 @@ # # General pooler settings [general] - # What IP to run on, 0.0.0.0 means accessible from everywhere. host = "0.0.0.0" # Port to run on, same as PgBouncer used in this example. port = 6432 -# How many connections to allocate per server. -pool_size = 15 - -# Pool mode (see PgBouncer docs for more). -# session: one server connection per connected client -# transaction: one server connection per client transaction -pool_mode = "transaction" - # How long to wait before aborting a server connection (ms). connect_timeout = 5000 @@ -27,7 +18,7 @@ connect_timeout = 5000 healthcheck_timeout = 1000 # For how long to ban a server if it fails a health check (seconds). -ban_time = 60 # Seconds +ban_time = 60 # seconds # Reload config automatically if it changes. autoreload = false @@ -36,50 +27,20 @@ autoreload = false # tls_certificate = "server.cert" # tls_private_key = "server.key" -# -# User to use for authentication against the server. -[user] -name = "sharding_user" -password = "sharding_user" - - -# -# Shards in the cluster -[shards] - -# Shard 0 -[shards.0] - -# [ host, port, role ] -servers = [ - [ "127.0.0.1", 5432, "primary" ], - [ "localhost", 5432, "replica" ], - # [ "127.0.1.1", 5432, "replica" ], -] -# Database name (e.g. "postgres") -database = "shard0" - -[shards.1] -# [ host, port, role ] -servers = [ - [ "127.0.0.1", 5432, "primary" ], - [ "localhost", 5432, "replica" ], - # [ "127.0.1.1", 5432, "replica" ], -] -database = "shard1" +# Credentials to access the virtual administrative database (pgbouncer or pgcat) +# Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. +admin_username = "user" +admin_password = "pass" -[shards.2] -# [ host, port, role ] -servers = [ - [ "127.0.0.1", 5432, "primary" ], - [ "localhost", 5432, "replica" ], - # [ "127.0.1.1", 5432, "replica" ], -] -database = "shard2" - - -# Settings for our query routing layer. -[query_router] +# pool +# configs are structured as pool. +# the pool_name is what clients use as database name when connecting +# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded" +[pools.sharded] +# Pool mode (see PgBouncer docs for more). +# session: one server connection per connected client +# transaction: one server connection per client transaction +pool_mode = "transaction" # If the client doesn't specify, route traffic to # this role by default. @@ -89,7 +50,6 @@ database = "shard2" # primary: all queries go to the primary unless otherwise specified. default_role = "any" - # Query parser. If enabled, we'll attempt to parse # every incoming query to determine if it's a read or a write. # If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, @@ -110,3 +70,61 @@ primary_reads_enabled = true # sha1: A hashing function based on SHA1 # sharding_function = "pg_bigint_hash" + +# Credentials for users that may connect to this cluster +[pools.sharded.users.0] +username = "sharding_user" +password = "sharding_user" +# Maximum number of server connections that can be established for this user +# The maximum number of connection from a single Pgcat process to any database in the cluster +# is the sum of pool_size across all users. +pool_size = 9 + +[pools.sharded.users.1] +username = "other_user" +password = "other_user" +pool_size = 21 + +# Shard 0 +[pools.sharded.shards.0] +# [ host, port, role ] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ] +] +# Database name (e.g. "postgres") +database = "shard0" + +[pools.sharded.shards.1] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ], +] +database = "shard1" + +[pools.sharded.shards.2] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ], +] +database = "shard2" + + +[pools.simple_db] +pool_mode = "session" +default_role = "primary" +query_parser_enabled = true +primary_reads_enabled = true +sharding_function = "pg_bigint_hash" + +[pools.simple_db.users.0] +username = "simple_user" +password = "simple_user" +pool_size = 5 + +[pools.simple_db.shards.0] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ] +] +database = "some_db" diff --git a/src/admin.rs b/src/admin.rs index 74acf1512..163227db9 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -3,10 +3,10 @@ use bytes::{Buf, BufMut, BytesMut}; use log::{info, trace}; use std::collections::HashMap; -use crate::config::{get_config, reload_config}; +use crate::config::{get_config, reload_config, VERSION}; use crate::errors::Error; use crate::messages::*; -use crate::pool::ConnectionPool; +use crate::pool::get_all_pools; use crate::stats::get_stats; use crate::ClientServerMap; @@ -14,7 +14,6 @@ use crate::ClientServerMap; pub async fn handle_admin( stream: &mut T, mut query: BytesMut, - pool: ConnectionPool, client_server_map: ClientServerMap, ) -> Result<(), Error> where @@ -35,7 +34,7 @@ where if query.starts_with("SHOW STATS") { trace!("SHOW STATS"); - show_stats(stream, &pool).await + show_stats(stream).await } else if query.starts_with("RELOAD") { trace!("RELOAD"); reload(stream, client_server_map).await @@ -44,13 +43,13 @@ where show_config(stream).await } else if query.starts_with("SHOW DATABASES") { trace!("SHOW DATABASES"); - show_databases(stream, &pool).await + show_databases(stream).await } else if query.starts_with("SHOW POOLS") { trace!("SHOW POOLS"); - show_pools(stream, &pool).await + show_pools(stream).await } else if query.starts_with("SHOW LISTS") { trace!("SHOW LISTS"); - show_lists(stream, &pool).await + show_lists(stream).await } else if query.starts_with("SHOW VERSION") { trace!("SHOW VERSION"); show_version(stream).await @@ -63,7 +62,7 @@ where } /// Column-oriented statistics. -async fn show_lists(stream: &mut T, pool: &ConnectionPool) -> Result<(), Error> +async fn show_lists(stream: &mut T) -> Result<(), Error> where T: tokio::io::AsyncWrite + std::marker::Unpin, { @@ -71,17 +70,20 @@ where let columns = vec![("list", DataType::Text), ("items", DataType::Int4)]; + let mut users = 1; + let mut databases = 1; + for (_, pool) in get_all_pools() { + databases += pool.databases(); + users += 1; // One user per pool + } let mut res = BytesMut::new(); res.put(row_description(&columns)); res.put(data_row(&vec![ "databases".to_string(), - (pool.databases() + 1).to_string(), // see comment below + databases.to_string(), ])); - res.put(data_row(&vec!["users".to_string(), "1".to_string()])); - res.put(data_row(&vec![ - "pools".to_string(), - (pool.databases() + 1).to_string(), // +1 for the pgbouncer admin db pool which isn't real - ])); // but admin tools that work with pgbouncer want this + res.put(data_row(&vec!["users".to_string(), users.to_string()])); + res.put(data_row(&vec!["pools".to_string(), databases.to_string()])); res.put(data_row(&vec![ "free_clients".to_string(), stats @@ -140,7 +142,7 @@ where let mut res = BytesMut::new(); res.put(row_description(&vec![("version", DataType::Text)])); - res.put(data_row(&vec!["PgCat 0.1.0".to_string()])); + res.put(data_row(&vec![format!("PgCat {}", VERSION).to_string()])); res.put(command_complete("SHOW")); res.put_u8(b'Z'); @@ -151,12 +153,11 @@ where } /// Show utilization of connection pools for each shard and replicas. -async fn show_pools(stream: &mut T, pool: &ConnectionPool) -> Result<(), Error> +async fn show_pools(stream: &mut T) -> Result<(), Error> where T: tokio::io::AsyncWrite + std::marker::Unpin, { let stats = get_stats(); - let config = get_config(); let columns = vec![ ("database", DataType::Text), @@ -176,24 +177,26 @@ where let mut res = BytesMut::new(); res.put(row_description(&columns)); - - for shard in 0..pool.shards() { - for server in 0..pool.servers(shard) { - let address = pool.address(shard, server); - let stats = match stats.get(&address.id) { - Some(stats) => stats.clone(), - None => HashMap::new(), - }; - - let mut row = vec![address.name(), config.user.name.clone()]; - - for column in &columns[2..columns.len() - 1] { - let value = stats.get(column.0).unwrap_or(&0).to_string(); - row.push(value); + for (_, pool) in get_all_pools() { + let pool_config = &pool.settings; + for shard in 0..pool.shards() { + for server in 0..pool.servers(shard) { + let address = pool.address(shard, server); + let stats = match stats.get(&address.id) { + Some(stats) => stats.clone(), + None => HashMap::new(), + }; + + let mut row = vec![address.name(), pool_config.user.username.clone()]; + + for column in &columns[2..columns.len() - 1] { + let value = stats.get(column.0).unwrap_or(&0).to_string(); + row.push(value); + } + + row.push(pool_config.pool_mode.to_string()); + res.put(data_row(&row)); } - - row.push(config.general.pool_mode.to_string()); - res.put(data_row(&row)); } } @@ -208,12 +211,10 @@ where } /// Show shards and replicas. -async fn show_databases(stream: &mut T, pool: &ConnectionPool) -> Result<(), Error> +async fn show_databases(stream: &mut T) -> Result<(), Error> where T: tokio::io::AsyncWrite + std::marker::Unpin, { - let config = get_config(); - // Columns let columns = vec![ ("name", DataType::Text), @@ -235,31 +236,33 @@ where res.put(row_description(&columns)); - for shard in 0..pool.shards() { - let database_name = &config.shards[&shard.to_string()].database; - - for server in 0..pool.servers(shard) { - let address = pool.address(shard, server); - let pool_state = pool.pool_state(shard, server); - - res.put(data_row(&vec![ - address.name(), // name - address.host.to_string(), // host - address.port.to_string(), // port - database_name.to_string(), // database - config.user.name.to_string(), // force_user - config.general.pool_size.to_string(), // pool_size - "0".to_string(), // min_pool_size - "0".to_string(), // reserve_pool - config.general.pool_mode.to_string(), // pool_mode - config.general.pool_size.to_string(), // max_connections - pool_state.connections.to_string(), // current_connections - "0".to_string(), // paused - "0".to_string(), // disabled - ])); + for (_, pool) in get_all_pools() { + let pool_config = pool.settings.clone(); + for shard in 0..pool.shards() { + let database_name = &pool_config.shards[&shard.to_string()].database; + + for server in 0..pool.servers(shard) { + let address = pool.address(shard, server); + let pool_state = pool.pool_state(shard, server); + + res.put(data_row(&vec![ + address.name(), // name + address.host.to_string(), // host + address.port.to_string(), // port + database_name.to_string(), // database + pool_config.user.username.to_string(), // force_user + pool_config.user.pool_size.to_string(), // pool_size + "0".to_string(), // min_pool_size + "0".to_string(), // reserve_pool + pool_config.pool_mode.to_string(), // pool_mode + pool_config.user.pool_size.to_string(), // max_connections + pool_state.connections.to_string(), // current_connections + "0".to_string(), // paused + "0".to_string(), // disabled + ])); + } } } - res.put(command_complete("SHOW")); // ReadyForQuery @@ -349,7 +352,7 @@ where } /// Show shard and replicas statistics. -async fn show_stats(stream: &mut T, pool: &ConnectionPool) -> Result<(), Error> +async fn show_stats(stream: &mut T) -> Result<(), Error> where T: tokio::io::AsyncWrite + std::marker::Unpin, { @@ -375,21 +378,23 @@ where let mut res = BytesMut::new(); res.put(row_description(&columns)); - for shard in 0..pool.shards() { - for server in 0..pool.servers(shard) { - let address = pool.address(shard, server); - let stats = match stats.get(&address.id) { - Some(stats) => stats.clone(), - None => HashMap::new(), - }; + for (_, pool) in get_all_pools() { + for shard in 0..pool.shards() { + for server in 0..pool.servers(shard) { + let address = pool.address(shard, server); + let stats = match stats.get(&address.id) { + Some(stats) => stats.clone(), + None => HashMap::new(), + }; - let mut row = vec![address.name()]; + let mut row = vec![address.name()]; - for column in &columns[1..] { - row.push(stats.get(column.0).unwrap_or(&0).to_string()); - } + for column in &columns[1..] { + row.push(stats.get(column.0).unwrap_or(&0).to_string()); + } - res.put(data_row(&row)); + res.put(data_row(&row)); + } } } diff --git a/src/client.rs b/src/client.rs index 05895b6d6..4f32d0af9 100644 --- a/src/client.rs +++ b/src/client.rs @@ -10,7 +10,7 @@ use crate::config::get_config; use crate::constants::*; use crate::errors::Error; use crate::messages::*; -use crate::pool::{get_pool, ClientServerMap}; +use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; use crate::stats::{get_reporter, Reporter}; @@ -71,6 +71,8 @@ pub struct Client { /// Last server process id we talked to. last_server_id: Option, + + target_pool: ConnectionPool, } /// Client entrypoint. @@ -258,11 +260,25 @@ where client_server_map: ClientServerMap, ) -> Result, Error> { let config = get_config(); - let transaction_mode = config.general.pool_mode == "transaction"; let stats = get_reporter(); trace!("Got StartupMessage"); let parameters = parse_startup(bytes.clone())?; + let database = match parameters.get("database") { + Some(db) => db, + None => return Err(Error::ClientError), + }; + + let user = match parameters.get("user") { + Some(user) => user, + None => return Err(Error::ClientError), + }; + + let admin = ["pgcat", "pgbouncer"] + .iter() + .filter(|db| *db == &database) + .count() + == 1; // Generate random backend ID and secret key let process_id: i32 = rand::random(); @@ -295,33 +311,57 @@ where Err(_) => return Err(Error::SocketError), }; - // Compare server and client hashes. - let password_hash = md5_hash_password(&config.user.name, &config.user.password, &salt); + let mut target_pool: ConnectionPool = ConnectionPool::default(); + let mut transaction_mode = false; + + if admin { + let correct_user = config.general.admin_username.as_str(); + let correct_password = config.general.admin_password.as_str(); + + // Compare server and client hashes. + let password_hash = md5_hash_password(correct_user, correct_password, &salt); + if password_hash != password_response { + debug!("Password authentication failed"); + wrong_password(&mut write, user).await?; + return Err(Error::ClientError); + } + } else { + target_pool = match get_pool(database.clone(), user.clone()) { + Some(pool) => pool, + None => { + error_response( + &mut write, + &format!( + "No pool configured for database: {:?}, user: {:?}", + database, user + ), + ) + .await?; + return Err(Error::ClientError); + } + }; + transaction_mode = target_pool.settings.pool_mode == "transaction"; + + // Compare server and client hashes. + let correct_password = target_pool.settings.user.password.as_str(); + let password_hash = md5_hash_password(user, correct_password, &salt); - if password_hash != password_response { - debug!("Password authentication failed"); - wrong_password(&mut write, &config.user.name).await?; - return Err(Error::ClientError); + if password_hash != password_response { + debug!("Password authentication failed"); + wrong_password(&mut write, user).await?; + return Err(Error::ClientError); + } } debug!("Password authentication successful"); auth_ok(&mut write).await?; - write_all(&mut write, get_pool().server_info()).await?; + write_all(&mut write, target_pool.server_info()).await?; backend_key_data(&mut write, process_id, secret_key).await?; ready_for_query(&mut write).await?; trace!("Startup OK"); - let database = parameters - .get("database") - .unwrap_or(parameters.get("user").unwrap()); - let admin = ["pgcat", "pgbouncer"] - .iter() - .filter(|db| *db == &database) - .count() - == 1; - // Split the read and write streams // so we can control buffering. @@ -335,11 +375,12 @@ where process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, - parameters: parameters, + parameters: parameters.clone(), stats: stats, admin: admin, last_address_id: None, last_server_id: None, + target_pool: target_pool, }); } @@ -353,26 +394,22 @@ where ) -> Result, Error> { let process_id = bytes.get_i32(); let secret_key = bytes.get_i32(); - - let config = get_config(); - let transaction_mode = config.general.pool_mode == "transaction"; - let stats = get_reporter(); - return Ok(Client { read: BufReader::new(read), write: write, addr, buffer: BytesMut::with_capacity(8196), cancel_mode: true, - transaction_mode: transaction_mode, + transaction_mode: false, process_id: process_id, secret_key: secret_key, client_server_map: client_server_map, parameters: HashMap::new(), - stats: stats, + stats: get_reporter(), admin: false, last_address_id: None, last_server_id: None, + target_pool: ConnectionPool::default(), }); } @@ -410,7 +447,7 @@ where // The query router determines where the query is going to go, // e.g. primary, replica, which shard. - let mut query_router = QueryRouter::new(); + let mut query_router = QueryRouter::new(self.target_pool.clone()); let mut round_robin = 0; // Our custom protocol loop. @@ -432,7 +469,7 @@ where // Get a pool instance referenced by the most up-to-date // pointer. This ensures we always read the latest config // when starting a query. - let mut pool = get_pool(); + let mut pool = self.target_pool.clone(); // Avoid taking a server if the client just wants to disconnect. if message[0] as char == 'X' { @@ -443,13 +480,7 @@ where // Handle admin database queries. if self.admin { debug!("Handling admin command"); - handle_admin( - &mut self.write, - message, - pool.clone(), - self.client_server_map.clone(), - ) - .await?; + handle_admin(&mut self.write, message, self.client_server_map.clone()).await?; continue; } diff --git a/src/config.rs b/src/config.rs index da59d2ae1..d660fdcfa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,6 +4,7 @@ use log::{error, info}; use once_cell::sync::Lazy; use serde_derive::Deserialize; use std::collections::{HashMap, HashSet}; +use std::hash::Hash; use std::path::Path; use std::sync::Arc; use tokio::fs::File; @@ -14,6 +15,8 @@ use crate::errors::Error; use crate::tls::{load_certs, load_keys}; use crate::{ClientServerMap, ConnectionPool}; +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + /// Globally available configuration. static CONFIG: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Config::default())); @@ -58,6 +61,7 @@ pub struct Address { pub host: String, pub port: String, pub shard: usize, + pub database: String, pub role: Role, pub replica_number: usize, } @@ -70,6 +74,7 @@ impl Default for Address { port: String::from("5432"), shard: 0, replica_number: 0, + database: String::from("database"), role: Role::Replica, } } @@ -79,9 +84,12 @@ impl Address { /// Address name (aka database) used in `SHOW STATS`, `SHOW DATABASES`, and `SHOW POOLS`. pub fn name(&self) -> String { match self.role { - Role::Primary => format!("shard_{}_primary", self.shard), + Role::Primary => format!("{}_shard_{}_primary", self.database, self.shard), - Role::Replica => format!("shard_{}_replica_{}", self.shard, self.replica_number), + Role::Replica => format!( + "{}_shard_{}_replica_{}", + self.database, self.shard, self.replica_number + ), } } } @@ -89,15 +97,17 @@ impl Address { /// PostgreSQL user. #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Deserialize, Debug)] pub struct User { - pub name: String, + pub username: String, pub password: String, + pub pool_size: u32, } impl Default for User { fn default() -> User { User { - name: String::from("postgres"), + username: String::from("postgres"), password: String::new(), + pool_size: 15, } } } @@ -107,14 +117,14 @@ impl Default for User { pub struct General { pub host: String, pub port: i16, - pub pool_size: u32, - pub pool_mode: String, pub connect_timeout: u64, pub healthcheck_timeout: u64, pub ban_time: i64, pub autoreload: bool, pub tls_certificate: Option, pub tls_private_key: Option, + pub admin_username: String, + pub admin_password: String, } impl Default for General { @@ -122,14 +132,37 @@ impl Default for General { General { host: String::from("localhost"), port: 5432, - pool_size: 15, - pool_mode: String::from("transaction"), connect_timeout: 5000, healthcheck_timeout: 1000, ban_time: 60, autoreload: false, tls_certificate: None, tls_private_key: None, + admin_username: String::from("admin"), + admin_password: String::from("admin"), + } + } +} +#[derive(Deserialize, Debug, Clone, PartialEq)] +pub struct Pool { + pub pool_mode: String, + pub shards: HashMap, + pub users: HashMap, + pub default_role: String, + pub query_parser_enabled: bool, + pub primary_reads_enabled: bool, + pub sharding_function: String, +} +impl Default for Pool { + fn default() -> Pool { + Pool { + pool_mode: String::from("transaction"), + shards: HashMap::from([(String::from("1"), Shard::default())]), + users: HashMap::default(), + default_role: String::from("any"), + query_parser_enabled: false, + primary_reads_enabled: true, + sharding_function: "pg_bigint_hash".to_string(), } } } @@ -137,8 +170,8 @@ impl Default for General { /// Shard configuration. #[derive(Deserialize, Debug, Clone, PartialEq)] pub struct Shard { - pub servers: Vec<(String, u16, String)>, pub database: String, + pub servers: Vec<(String, u16, String)>, } impl Default for Shard { @@ -150,26 +183,6 @@ impl Default for Shard { } } -/// Query Router configuration. -#[derive(Deserialize, Debug, Clone, PartialEq)] -pub struct QueryRouter { - pub default_role: String, - pub query_parser_enabled: bool, - pub primary_reads_enabled: bool, - pub sharding_function: String, -} - -impl Default for QueryRouter { - fn default() -> QueryRouter { - QueryRouter { - default_role: String::from("any"), - query_parser_enabled: false, - primary_reads_enabled: true, - sharding_function: "pg_bigint_hash".to_string(), - } - } -} - fn default_path() -> String { String::from("pgcat.toml") } @@ -181,9 +194,7 @@ pub struct Config { pub path: String, pub general: General, - pub user: User, - pub shards: HashMap, - pub query_router: QueryRouter, + pub pools: HashMap, } impl Default for Config { @@ -191,26 +202,58 @@ impl Default for Config { Config { path: String::from("pgcat.toml"), general: General::default(), - user: User::default(), - shards: HashMap::from([(String::from("1"), Shard::default())]), - query_router: QueryRouter::default(), + pools: HashMap::default(), } } } impl From<&Config> for std::collections::HashMap { fn from(config: &Config) -> HashMap { - HashMap::from([ + let mut r: Vec<(String, String)> = config + .pools + .iter() + .flat_map(|(pool_name, pool)| { + [ + ( + format!("pools.{}.pool_mode", pool_name), + pool.pool_mode.clone(), + ), + ( + format!("pools.{}.primary_reads_enabled", pool_name), + pool.primary_reads_enabled.to_string(), + ), + ( + format!("pools.{}.query_parser_enabled", pool_name), + pool.query_parser_enabled.to_string(), + ), + ( + format!("pools.{}.default_role", pool_name), + pool.default_role.clone(), + ), + ( + format!("pools.{}.sharding_function", pool_name), + pool.sharding_function.clone(), + ), + ( + format!("pools.{:?}.shard_count", pool_name), + pool.shards.len().to_string(), + ), + ( + format!("pools.{:?}.users", pool_name), + pool.users + .iter() + .map(|(_username, user)| &user.username) + .cloned() + .collect::>() + .join(", "), + ), + ] + }) + .collect(); + + let mut static_settings = vec![ ("host".to_string(), config.general.host.to_string()), ("port".to_string(), config.general.port.to_string()), - ( - "pool_size".to_string(), - config.general.pool_size.to_string(), - ), - ( - "pool_mode".to_string(), - config.general.pool_mode.to_string(), - ), ( "connect_timeout".to_string(), config.general.connect_timeout.to_string(), @@ -220,42 +263,22 @@ impl From<&Config> for std::collections::HashMap { config.general.healthcheck_timeout.to_string(), ), ("ban_time".to_string(), config.general.ban_time.to_string()), - ( - "default_role".to_string(), - config.query_router.default_role.to_string(), - ), - ( - "query_parser_enabled".to_string(), - config.query_router.query_parser_enabled.to_string(), - ), - ( - "primary_reads_enabled".to_string(), - config.query_router.primary_reads_enabled.to_string(), - ), - ( - "sharding_function".to_string(), - config.query_router.sharding_function.to_string(), - ), - ]) + ]; + + r.append(&mut static_settings); + return r.iter().cloned().collect(); } } impl Config { /// Print current configuration. pub fn show(&self) { - info!("Pool size: {}", self.general.pool_size); - info!("Pool mode: {}", self.general.pool_mode); info!("Ban time: {}s", self.general.ban_time); info!( "Healthcheck timeout: {}ms", self.general.healthcheck_timeout ); info!("Connection timeout: {}ms", self.general.connect_timeout); - info!("Sharding function: {}", self.query_router.sharding_function); - info!("Primary reads: {}", self.query_router.primary_reads_enabled); - info!("Query router: {}", self.query_router.query_parser_enabled); - info!("Number of shards: {}", self.shards.len()); - match self.general.tls_certificate.clone() { Some(tls_certificate) => { info!("TLS certificate: {}", tls_certificate); @@ -274,6 +297,25 @@ impl Config { info!("TLS support is disabled"); } }; + + for (pool_name, pool_config) in &self.pools { + info!("--- Settings for pool {} ---", pool_name); + info!( + "Pool size from all users: {}", + pool_config + .users + .iter() + .map(|(_, user_cfg)| user_cfg.pool_size) + .sum::() + .to_string() + ); + info!("Pool mode: {}", pool_config.pool_mode); + info!("Sharding function: {}", pool_config.sharding_function); + info!("Primary reads: {}", pool_config.primary_reads_enabled); + info!("Query router: {}", pool_config.query_parser_enabled); + info!("Number of shards: {}", pool_config.shards.len()); + info!("Number of users: {}", pool_config.users.len()); + } } } @@ -311,88 +353,6 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; - match config.query_router.sharding_function.as_ref() { - "pg_bigint_hash" => (), - "sha1" => (), - _ => { - error!( - "Supported sharding functions are: 'pg_bigint_hash', 'sha1', got: '{}'", - config.query_router.sharding_function - ); - return Err(Error::BadConfig); - } - }; - - // Quick config sanity check. - for shard in &config.shards { - // We use addresses as unique identifiers, - // let's make sure they are unique in the config as well. - let mut dup_check = HashSet::new(); - let mut primary_count = 0; - - match shard.0.parse::() { - Ok(_) => (), - Err(_) => { - error!( - "Shard '{}' is not a valid number, shards must be numbered starting at 0", - shard.0 - ); - return Err(Error::BadConfig); - } - }; - - if shard.1.servers.len() == 0 { - error!("Shard {} has no servers configured", shard.0); - return Err(Error::BadConfig); - } - - for server in &shard.1.servers { - dup_check.insert(server); - - // Check that we define only zero or one primary. - match server.2.as_ref() { - "primary" => primary_count += 1, - _ => (), - }; - - // Check role spelling. - match server.2.as_ref() { - "primary" => (), - "replica" => (), - _ => { - error!( - "Shard {} server role must be either 'primary' or 'replica', got: '{}'", - shard.0, server.2 - ); - return Err(Error::BadConfig); - } - }; - } - - if primary_count > 1 { - error!("Shard {} has more than on primary configured", &shard.0); - return Err(Error::BadConfig); - } - - if dup_check.len() != shard.1.servers.len() { - error!("Shard {} contains duplicate server configs", &shard.0); - return Err(Error::BadConfig); - } - } - - match config.query_router.default_role.as_ref() { - "any" => (), - "primary" => (), - "replica" => (), - other => { - error!( - "Query router default_role must be 'primary', 'replica', or 'any', got: '{}'", - other - ); - return Err(Error::BadConfig); - } - }; - // Validate TLS! match config.general.tls_certificate.clone() { Some(tls_certificate) => { @@ -424,6 +384,90 @@ pub async fn parse(path: &str) -> Result<(), Error> { None => (), }; + for (pool_name, pool) in &config.pools { + match pool.sharding_function.as_ref() { + "pg_bigint_hash" => (), + "sha1" => (), + _ => { + error!( + "Supported sharding functions are: 'pg_bigint_hash', 'sha1', got: '{}' in pool {} settings", + pool.sharding_function, + pool_name + ); + return Err(Error::BadConfig); + } + }; + + match pool.default_role.as_ref() { + "any" => (), + "primary" => (), + "replica" => (), + other => { + error!( + "Query router default_role must be 'primary', 'replica', or 'any', got: '{}'", + other + ); + return Err(Error::BadConfig); + } + }; + + for shard in &pool.shards { + // We use addresses as unique identifiers, + // let's make sure they are unique in the config as well. + let mut dup_check = HashSet::new(); + let mut primary_count = 0; + + match shard.0.parse::() { + Ok(_) => (), + Err(_) => { + error!( + "Shard '{}' is not a valid number, shards must be numbered starting at 0", + shard.0 + ); + return Err(Error::BadConfig); + } + }; + + if shard.1.servers.len() == 0 { + error!("Shard {} has no servers configured", shard.0); + return Err(Error::BadConfig); + } + + for server in &shard.1.servers { + dup_check.insert(server); + + // Check that we define only zero or one primary. + match server.2.as_ref() { + "primary" => primary_count += 1, + _ => (), + }; + + // Check role spelling. + match server.2.as_ref() { + "primary" => (), + "replica" => (), + _ => { + error!( + "Shard {} server role must be either 'primary' or 'replica', got: '{}'", + shard.0, server.2 + ); + return Err(Error::BadConfig); + } + }; + } + + if primary_count > 1 { + error!("Shard {} has more than on primary configured", &shard.0); + return Err(Error::BadConfig); + } + + if dup_check.len() != shard.1.servers.len() { + error!("Shard {} contains duplicate server configs", &shard.0); + return Err(Error::BadConfig); + } + } + } + config.path = path.to_string(); // Update the configuration globally. @@ -434,7 +478,6 @@ pub async fn parse(path: &str) -> Result<(), Error> { pub async fn reload_config(client_server_map: ClientServerMap) -> Result { let old_config = get_config(); - match parse(&old_config.path).await { Ok(()) => (), Err(err) => { @@ -442,11 +485,10 @@ pub async fn reload_config(client_server_map: ClientServerMap) -> Result>>>; pub type ClientServerMap = Arc>>; - +pub type PoolMap = HashMap<(String, String), ConnectionPool>; /// The connection pool, globally available. /// This is atomic and safe and read-optimized. /// The pool is recreated dynamically when the config is reloaded. -pub static POOL: Lazy> = - Lazy::new(|| ArcSwap::from_pointee(ConnectionPool::default())); +pub static POOLS: Lazy> = Lazy::new(|| ArcSwap::from_pointee(HashMap::default())); + +#[derive(Clone, Debug)] +pub struct PoolSettings { + pub pool_mode: String, + pub shards: HashMap, + pub user: User, + pub default_role: String, + pub query_parser_enabled: bool, + pub primary_reads_enabled: bool, + pub sharding_function: String, +} +impl Default for PoolSettings { + fn default() -> PoolSettings { + PoolSettings { + pool_mode: String::from("transaction"), + shards: HashMap::from([(String::from("1"), Shard::default())]), + user: User::default(), + default_role: String::from("any"), + query_parser_enabled: false, + primary_reads_enabled: true, + sharding_function: "pg_bigint_hash".to_string(), + } + } +} /// The globally accessible connection pool. #[derive(Clone, Debug, Default)] @@ -46,107 +70,124 @@ pub struct ConnectionPool { /// clients on startup. We pre-connect to all shards and replicas /// on pool creation and save the K messages here. server_info: BytesMut, + + pub settings: PoolSettings, } impl ConnectionPool { /// Construct the connection pool from the configuration. pub async fn from_config(client_server_map: ClientServerMap) -> Result<(), Error> { - let reporter = get_reporter(); let config = get_config(); + let mut new_pools = PoolMap::default(); - let mut shards = Vec::new(); - let mut addresses = Vec::new(); - let mut banlist = Vec::new(); let mut address_id = 0; - let mut shard_ids = config - .shards - .clone() - .into_keys() - .map(|x| x.to_string()) - .collect::>(); - - // Sort by shard number to ensure consistency. - shard_ids.sort_by_key(|k| k.parse::().unwrap()); - - for shard_idx in shard_ids { - let shard = &config.shards[&shard_idx]; - let mut pools = Vec::new(); - let mut servers = Vec::new(); - let mut replica_number = 0; - - for server in shard.servers.iter() { - let role = match server.2.as_ref() { - "primary" => Role::Primary, - "replica" => Role::Replica, - _ => { - error!("Config error: server role can be 'primary' or 'replica', have: '{}'. Defaulting to 'replica'.", server.2); - Role::Replica - } - }; - - let address = Address { - id: address_id, - host: server.0.clone(), - port: server.1.to_string(), - role: role, - replica_number, - shard: shard_idx.parse::().unwrap(), - }; + for (pool_name, pool_config) in &config.pools { + for (_user_index, user_info) in &pool_config.users { + let mut shards = Vec::new(); + let mut addresses = Vec::new(); + let mut banlist = Vec::new(); + let mut shard_ids = pool_config + .shards + .clone() + .into_keys() + .map(|x| x.to_string()) + .collect::>(); + + // Sort by shard number to ensure consistency. + shard_ids.sort_by_key(|k| k.parse::().unwrap()); + + for shard_idx in shard_ids { + let shard = &pool_config.shards[&shard_idx]; + let mut pools = Vec::new(); + let mut servers = Vec::new(); + let mut replica_number = 0; + + for server in shard.servers.iter() { + let role = match server.2.as_ref() { + "primary" => Role::Primary, + "replica" => Role::Replica, + _ => { + error!("Config error: server role can be 'primary' or 'replica', have: '{}'. Defaulting to 'replica'.", server.2); + Role::Replica + } + }; + + let address = Address { + id: address_id, + database: pool_name.clone(), + host: server.0.clone(), + port: server.1.to_string(), + role: role, + replica_number, + shard: shard_idx.parse::().unwrap(), + }; + + address_id += 1; + + if role == Role::Replica { + replica_number += 1; + } + + let manager = ServerPool::new( + address.clone(), + user_info.clone(), + &shard.database, + client_server_map.clone(), + get_reporter(), + ); - address_id += 1; + let pool = Pool::builder() + .max_size(user_info.pool_size) + .connection_timeout(std::time::Duration::from_millis( + config.general.connect_timeout, + )) + .test_on_check_out(false) + .build(manager) + .await + .unwrap(); + + pools.push(pool); + servers.push(address); + } - if role == Role::Replica { - replica_number += 1; + shards.push(pools); + addresses.push(servers); + banlist.push(HashMap::new()); } - let manager = ServerPool::new( - address.clone(), - config.user.clone(), - &shard.database, - client_server_map.clone(), - reporter.clone(), - ); - - let pool = Pool::builder() - .max_size(config.general.pool_size) - .connection_timeout(std::time::Duration::from_millis( - config.general.connect_timeout, - )) - .test_on_check_out(false) - .build(manager) - .await - .unwrap(); - - pools.push(pool); - servers.push(address); - } - - shards.push(pools); - addresses.push(servers); - banlist.push(HashMap::new()); - } - - assert_eq!(shards.len(), addresses.len()); - - let mut pool = ConnectionPool { - databases: shards, - addresses: addresses, - banlist: Arc::new(RwLock::new(banlist)), - stats: reporter, - server_info: BytesMut::new(), - }; + assert_eq!(shards.len(), addresses.len()); + + let mut pool = ConnectionPool { + databases: shards, + addresses: addresses, + banlist: Arc::new(RwLock::new(banlist)), + stats: get_reporter(), + server_info: BytesMut::new(), + settings: PoolSettings { + pool_mode: pool_config.pool_mode.clone(), + shards: pool_config.shards.clone(), + user: user_info.clone(), + default_role: pool_config.default_role.clone(), + query_parser_enabled: pool_config.query_parser_enabled.clone(), + primary_reads_enabled: pool_config.primary_reads_enabled, + sharding_function: pool_config.sharding_function.clone(), + }, + }; - // Connect to the servers to make sure pool configuration is valid - // before setting it globally. - match pool.validate().await { - Ok(_) => (), - Err(err) => { - error!("Could not validate connection pool: {:?}", err); - return Err(err); + // Connect to the servers to make sure pool configuration is valid + // before setting it globally. + match pool.validate().await { + Ok(_) => (), + Err(err) => { + error!("Could not validate connection pool: {:?}", err); + return Err(err); + } + }; + new_pools.insert((pool_name.clone(), user_info.username.clone()), pool); } - }; + } - POOL.store(Arc::new(pool.clone())); + POOLS.store(Arc::new(new_pools.clone())); Ok(()) } @@ -474,7 +515,7 @@ impl ManageConnection for ServerPool { info!( "Creating a new connection to {:?} using user {:?}", self.address.name(), - self.user.name + self.user.username ); // Put a temporary process_id into the stats @@ -517,6 +558,20 @@ impl ManageConnection for ServerPool { } /// Get the connection pool -pub fn get_pool() -> ConnectionPool { - (*(*POOL.load())).clone() +pub fn get_pool(db: String, user: String) -> Option { + match get_all_pools().get(&(db, user)) { + Some(pool) => Some(pool.clone()), + None => None, + } +} + +pub fn get_number_of_addresses() -> usize { + get_all_pools() + .iter() + .map(|(_, pool)| pool.databases()) + .sum() +} + +pub fn get_all_pools() -> HashMap<(String, String), ConnectionPool> { + return (*(*POOLS.load())).clone(); } diff --git a/src/query_router.rs b/src/query_router.rs index 98a47702b..d597b81ee 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -8,7 +8,8 @@ use sqlparser::ast::Statement::{Query, StartTransaction}; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; -use crate::config::{get_config, Role}; +use crate::config::Role; +use crate::pool::{ConnectionPool, PoolSettings}; use crate::sharding::{Sharder, ShardingFunction}; /// Regexes used to parse custom commands. @@ -53,6 +54,8 @@ pub struct QueryRouter { /// Include the primary into the replica pool for reads. primary_reads_enabled: bool, + + pool_settings: PoolSettings, } impl QueryRouter { @@ -88,14 +91,13 @@ impl QueryRouter { } /// Create a new instance of the query router. Each client gets its own. - pub fn new() -> QueryRouter { - let config = get_config(); - + pub fn new(target_pool: ConnectionPool) -> QueryRouter { QueryRouter { active_shard: None, active_role: None, - query_parser_enabled: config.query_router.query_parser_enabled, - primary_reads_enabled: config.query_router.primary_reads_enabled, + query_parser_enabled: target_pool.settings.query_parser_enabled, + primary_reads_enabled: target_pool.settings.primary_reads_enabled, + pool_settings: target_pool.settings, } } @@ -130,15 +132,13 @@ impl QueryRouter { return None; } - let config = get_config(); - - let sharding_function = match config.query_router.sharding_function.as_ref() { + let sharding_function = match self.pool_settings.sharding_function.as_ref() { "pg_bigint_hash" => ShardingFunction::PgBigintHash, "sha1" => ShardingFunction::Sha1, _ => unreachable!(), }; - let default_server_role = match config.query_router.default_role.as_ref() { + let default_server_role = match self.pool_settings.default_role.as_ref() { "any" => None, "primary" => Some(Role::Primary), "replica" => Some(Role::Replica), @@ -196,7 +196,7 @@ impl QueryRouter { match command { Command::SetShardingKey => { - let sharder = Sharder::new(config.shards.len(), sharding_function); + let sharder = Sharder::new(self.pool_settings.shards.len(), sharding_function); let shard = sharder.shard(value.parse::().unwrap()); self.active_shard = Some(shard); value = shard.to_string(); @@ -204,7 +204,7 @@ impl QueryRouter { Command::SetShard => { self.active_shard = match value.to_ascii_uppercase().as_ref() { - "ANY" => Some(rand::random::() % config.shards.len()), + "ANY" => Some(rand::random::() % self.pool_settings.shards.len()), _ => Some(value.parse::().unwrap()), }; } @@ -233,7 +233,7 @@ impl QueryRouter { "default" => { self.active_role = default_server_role; - self.query_parser_enabled = config.query_router.query_parser_enabled; + self.query_parser_enabled = self.query_parser_enabled; self.active_role } @@ -250,7 +250,7 @@ impl QueryRouter { self.primary_reads_enabled = false; } else if value == "default" { debug!("Setting primary reads to default"); - self.primary_reads_enabled = config.query_router.primary_reads_enabled; + self.primary_reads_enabled = self.pool_settings.primary_reads_enabled; } } @@ -370,7 +370,7 @@ mod test { #[test] fn test_defaults() { QueryRouter::setup(); - let qr = QueryRouter::new(); + let qr = QueryRouter::new(ConnectionPool::default()); assert_eq!(qr.role(), None); } @@ -378,7 +378,7 @@ mod test { #[test] fn test_infer_role_replica() { QueryRouter::setup(); - let mut qr = QueryRouter::new(); + let mut qr = QueryRouter::new(ConnectionPool::default()); assert!(qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")) != None); assert_eq!(qr.query_parser_enabled(), true); @@ -402,7 +402,7 @@ mod test { #[test] fn test_infer_role_primary() { QueryRouter::setup(); - let mut qr = QueryRouter::new(); + let mut qr = QueryRouter::new(ConnectionPool::default()); let queries = vec![ simple_query("UPDATE items SET name = 'pumpkin' WHERE id = 5"), @@ -421,7 +421,7 @@ mod test { #[test] fn test_infer_role_primary_reads_enabled() { QueryRouter::setup(); - let mut qr = QueryRouter::new(); + let mut qr = QueryRouter::new(ConnectionPool::default()); let query = simple_query("SELECT * FROM items WHERE id = 5"); assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO on")) != None); @@ -432,7 +432,7 @@ mod test { #[test] fn test_infer_role_parse_prepared() { QueryRouter::setup(); - let mut qr = QueryRouter::new(); + let mut qr = QueryRouter::new(ConnectionPool::default()); qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")); assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); @@ -523,15 +523,15 @@ mod test { #[test] fn test_try_execute_command() { QueryRouter::setup(); - let mut qr = QueryRouter::new(); + let mut qr = QueryRouter::new(ConnectionPool::default()); // SetShardingKey let query = simple_query("SET SHARDING KEY TO 13"); assert_eq!( qr.try_execute_command(query), - Some((Command::SetShardingKey, String::from("1"))) + Some((Command::SetShardingKey, String::from("0"))) ); - assert_eq!(qr.shard(), 1); + assert_eq!(qr.shard(), 0); // SetShard let query = simple_query("SET SHARD TO '1'"); @@ -600,7 +600,7 @@ mod test { #[test] fn test_enable_query_parser() { QueryRouter::setup(); - let mut qr = QueryRouter::new(); + let mut qr = QueryRouter::new(ConnectionPool::default()); let query = simple_query("SET SERVER ROLE TO 'auto'"); assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); diff --git a/src/server.rs b/src/server.rs index b20d15335..d21696f0b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -82,7 +82,7 @@ impl Server { trace!("Sending StartupMessage"); // StartupMessage - startup(&mut stream, &user.name, database).await?; + startup(&mut stream, &user.username, database).await?; let mut server_info = BytesMut::new(); let mut process_id: i32 = 0; @@ -127,7 +127,7 @@ impl Server { Err(_) => return Err(Error::SocketError), }; - md5_password(&mut stream, &user.name, &user.password, &salt[..]) + md5_password(&mut stream, &user.username, &user.password, &salt[..]) .await?; } diff --git a/src/stats.rs b/src/stats.rs index 59a03dc71..e01133951 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -6,7 +6,7 @@ use parking_lot::Mutex; use std::collections::HashMap; use tokio::sync::mpsc::{channel, Receiver, Sender}; -use crate::pool::get_pool; +use crate::pool::get_number_of_addresses; pub static REPORTER: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Reporter::default())); @@ -331,8 +331,8 @@ impl Collector { tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD / 15)); loop { interval.tick().await; - let addresses = get_pool().databases(); - for address_id in 0..addresses { + let address_count = get_number_of_addresses(); + for address_id in 0..address_count { let _ = tx.try_send(Event { name: EventName::UpdateStats, value: 0, @@ -349,8 +349,8 @@ impl Collector { tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD)); loop { interval.tick().await; - let addresses = get_pool().databases(); - for address_id in 0..addresses { + let address_count = get_number_of_addresses(); + for address_id in 0..address_count { let _ = tx.try_send(Event { name: EventName::UpdateAverages, value: 0, diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index 983619f0f..aaabe5e4e 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -15,7 +15,7 @@ port: 6432, username: 'sharding_user', password: 'sharding_user', - database: 'rails_dev', + database: 'sharded_db', application_name: 'testing_pgcat', prepared_statements: false, # Transaction mode advisory_locks: false # Same @@ -117,7 +117,7 @@ def down # Test evil clients def poorly_behaved_client - conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/rails_dev?application_name=testing_pgcat") + conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") conn.async_exec 'BEGIN' conn.async_exec 'SELECT 1' From 468888de16c8c9bb458167235d4546e113366b32 Mon Sep 17 00:00:00 2001 From: Pradeep Chhetri <30620077+chhetripradeep@users.noreply.github.com> Date: Thu, 28 Jul 2022 13:59:33 +0800 Subject: [PATCH 158/551] Minor fix for some stats (#97) --- .gitignore | 1 + src/stats.rs | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 50ccb252b..a4b784117 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +.idea /target *.deb diff --git a/src/stats.rs b/src/stats.rs index e01133951..83aa9cb5b 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -292,17 +292,18 @@ impl Collector { let stats_template = HashMap::from([ ("total_query_count", 0), - ("total_xact_count", 0), - ("total_sent", 0), + ("total_query_time", 0), ("total_received", 0), + ("total_sent", 0), + ("total_xact_count", 0), ("total_xact_time", 0), - ("total_query_time", 0), ("total_wait_time", 0), - ("avg_xact_time", 0), + ("avg_query_count", 0), ("avg_query_time", 0), - ("avg_xact_count", 0), - ("avg_sent", 0), ("avg_recv", 0), + ("avg_sent", 0), + ("avg_xact_count", 0), + ("avg_xact_time", 0), ("avg_wait_time", 0), ("maxwait_us", 0), ("maxwait", 0), @@ -506,9 +507,11 @@ impl Collector { // Calculate averages for stat in &[ "avg_query_count", - "avgxact_count", - "avg_sent", + "avg_query_time", "avg_recv", + "avg_sent", + "avg_xact_time", + "avg_xact_count", "avg_wait_time", ] { let total_name = match stat { From c0cff64f6ab6cbdee79e20c1ab0e229da83f299e Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 28 Jul 2022 17:42:04 -0500 Subject: [PATCH 159/551] Add Serialize trait to configs (#99) --- src/config.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/config.rs b/src/config.rs index d660fdcfa..4577bfa00 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,7 +2,7 @@ use arc_swap::ArcSwap; use log::{error, info}; use once_cell::sync::Lazy; -use serde_derive::Deserialize; +use serde_derive::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::hash::Hash; use std::path::Path; @@ -21,7 +21,7 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION"); static CONFIG: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Config::default())); /// Server role: primary or replica. -#[derive(Clone, PartialEq, Deserialize, Hash, std::cmp::Eq, Debug, Copy)] +#[derive(Clone, PartialEq, Serialize, Deserialize, Hash, std::cmp::Eq, Debug, Copy)] pub enum Role { Primary, Replica, @@ -95,7 +95,7 @@ impl Address { } /// PostgreSQL user. -#[derive(Clone, PartialEq, Hash, std::cmp::Eq, Deserialize, Debug)] +#[derive(Clone, PartialEq, Hash, std::cmp::Eq, Serialize, Deserialize, Debug)] pub struct User { pub username: String, pub password: String, @@ -113,7 +113,7 @@ impl Default for User { } /// General configuration. -#[derive(Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct General { pub host: String, pub port: i16, @@ -143,7 +143,7 @@ impl Default for General { } } } -#[derive(Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Pool { pub pool_mode: String, pub shards: HashMap, @@ -168,7 +168,7 @@ impl Default for Pool { } /// Shard configuration. -#[derive(Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Shard { pub database: String, pub servers: Vec<(String, u16, String)>, @@ -188,7 +188,7 @@ fn default_path() -> String { } /// Configuration wrapper. -#[derive(Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Config { #[serde(default = "default_path")] pub path: String, From 553975be22b8715bc7244c02986713f1e7b9fc64 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 29 Jul 2022 13:58:25 -0500 Subject: [PATCH 160/551] Slightly more light weight health check (#100) --- .circleci/pgcat.toml | 2 +- src/pool.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index eca8f6736..1c0c01040 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -14,7 +14,7 @@ port = 6432 # How long to wait before aborting a server connection (ms). connect_timeout = 100 -# How much time to give `SELECT 1` health check query to return with a result (ms). +# How much time to give the health check query to return with a result (ms). healthcheck_timeout = 100 # For how long to ban a server if it fails a health check (seconds). diff --git a/src/pool.rs b/src/pool.rs index eab333094..775c8d42b 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -328,7 +328,7 @@ impl ConnectionPool { match tokio::time::timeout( tokio::time::Duration::from_millis(healthcheck_timeout), - server.query("SELECT 1"), + server.query(";"), ) .await { From f6e3a22b587b3c51268532479b7cac3cade4a638 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sat, 30 Jul 2022 18:12:02 -0500 Subject: [PATCH 161/551] Avoid ValueAfterTable when serializing configs (#101) --- src/config.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 4577bfa00..f200f627f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -146,12 +146,12 @@ impl Default for General { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Pool { pub pool_mode: String, - pub shards: HashMap, - pub users: HashMap, pub default_role: String, pub query_parser_enabled: bool, pub primary_reads_enabled: bool, pub sharding_function: String, + pub shards: HashMap, + pub users: HashMap, } impl Default for Pool { fn default() -> Pool { @@ -190,6 +190,17 @@ fn default_path() -> String { /// Configuration wrapper. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Config { + // Serializer maintains the order of fields in the struct + // so we should always put simple fields before nested fields + // in all serializable structs to avoid ValueAfterTable errors + // These errors occur when the toml serializer is about to produce + // ambigous toml structure like the one below + // [main] + // field1_under_main = 1 + // field2_under_main = 2 + // [main.subconf] + // field1_under_subconf = 1 + // field3_under_main = 3 # This field will be interpreted as being under subconf and not under main #[serde(default = "default_path")] pub path: String, From 8ad30b11052a8fc454c589735e4a9ccb7eac8915 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sat, 30 Jul 2022 18:28:25 -0500 Subject: [PATCH 162/551] Add test for config Serializer (#102) * Add test for serializer * fmt --- src/config.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/config.rs b/src/config.rs index f200f627f..f1138f98b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -570,4 +570,10 @@ mod test { ); assert_eq!(get_config().pools["simple_db"].users["0"].pool_size, 5); } + + #[tokio::test] + async fn test_serialize_configs() { + parse("pgcat.toml").await.unwrap(); + print!("{}", toml::to_string(&get_config()).unwrap()); + } } From 5d5634f2a56e0c53c31e2d897397425a7ed32047 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sun, 31 Jul 2022 21:52:23 -0500 Subject: [PATCH 163/551] Send proper server parameters to clients using admin db (#103) * Send proper server parameters to clients using admin db * clean up * fix python test * build * Add python * missing & * debug ls * fix tests * fix tests * fix * Fix warning * Address comments --- .circleci/config.yml | 4 ++-- .circleci/run_tests.sh | 13 +++++++++++-- src/admin.rs | 12 ++++++++++++ src/client.rs | 23 +++++++++++++---------- src/messages.rs | 18 ++++++++++++++++++ tests/python/tests.py | 23 +++++++++++++++++------ tests/ruby/tests.rb | 13 +++++++++++++ 7 files changed, 86 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09fc893ee..e9bb12f06 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,9 +32,9 @@ jobs: command: "cargo fmt --check" - run: name: "Install dependencies" - command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev" + command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python" - run: - name: "Build" + name: "Build" command: "cargo build" - run: name: "Test" diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index c932a86ee..a6fb7909f 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -69,8 +69,17 @@ psql -U sharding_user -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_te cd tests/ruby && \ sudo gem install bundler && \ bundle install && \ - ruby tests.rb && \ -cd ../.. + ruby tests.rb +cd /home/circleci/project + +# +# Python tests +# +cd tests/python && \ + pip install -r requirements.txt && \ + python tests.py +cd /home/circleci/project + # Admin tests export PGPASSWORD=admin_pass diff --git a/src/admin.rs b/src/admin.rs index 163227db9..831ca0b6c 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -10,6 +10,18 @@ use crate::pool::get_all_pools; use crate::stats::get_stats; use crate::ClientServerMap; +pub fn generate_server_info_for_admin() -> BytesMut { + let mut server_info = BytesMut::new(); + + server_info.put(server_paramater_message("application_name", "")); + server_info.put(server_paramater_message("client_encoding", "UTF8")); + server_info.put(server_paramater_message("server_encoding", "UTF8")); + server_info.put(server_paramater_message("server_version", VERSION)); + server_info.put(server_paramater_message("DateStyle", "ISO, MDY")); + + return server_info; +} + /// Handle admin client. pub async fn handle_admin( stream: &mut T, diff --git a/src/client.rs b/src/client.rs index 4f32d0af9..1775ad22a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; use tokio::net::TcpStream; -use crate::admin::handle_admin; +use crate::admin::{generate_server_info_for_admin, handle_admin}; use crate::config::get_config; use crate::constants::*; use crate::errors::Error; @@ -311,10 +311,7 @@ where Err(_) => return Err(Error::SocketError), }; - let mut target_pool: ConnectionPool = ConnectionPool::default(); - let mut transaction_mode = false; - - if admin { + let (target_pool, transaction_mode, server_info) = if admin { let correct_user = config.general.admin_username.as_str(); let correct_password = config.general.admin_password.as_str(); @@ -325,8 +322,13 @@ where wrong_password(&mut write, user).await?; return Err(Error::ClientError); } + ( + ConnectionPool::default(), + false, + generate_server_info_for_admin(), + ) } else { - target_pool = match get_pool(database.clone(), user.clone()) { + let target_pool = match get_pool(database.clone(), user.clone()) { Some(pool) => pool, None => { error_response( @@ -340,8 +342,8 @@ where return Err(Error::ClientError); } }; - transaction_mode = target_pool.settings.pool_mode == "transaction"; - + let transaction_mode = target_pool.settings.pool_mode == "transaction"; + let server_info = target_pool.server_info(); // Compare server and client hashes. let correct_password = target_pool.settings.user.password.as_str(); let password_hash = md5_hash_password(user, correct_password, &salt); @@ -351,12 +353,13 @@ where wrong_password(&mut write, user).await?; return Err(Error::ClientError); } - } + (target_pool, transaction_mode, server_info) + }; debug!("Password authentication successful"); auth_ok(&mut write).await?; - write_all(&mut write, target_pool.server_info()).await?; + write_all(&mut write, server_info).await?; backend_key_data(&mut write, process_id, secret_key).await?; ready_for_query(&mut write).await?; diff --git a/src/messages.rs b/src/messages.rs index 89795c65e..ba22a5791 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -7,6 +7,7 @@ use tokio::net::TcpStream; use crate::errors::Error; use std::collections::HashMap; +use std::mem; /// Postgres data type mappings /// used in RowDescription ('T') message. @@ -498,3 +499,20 @@ where Ok(bytes) } + +pub fn server_paramater_message(key: &str, value: &str) -> BytesMut { + let mut server_info = BytesMut::new(); + + let null_byte_size = 1; + let len: usize = + mem::size_of::() + key.len() + null_byte_size + value.len() + null_byte_size; + + server_info.put_slice("S".as_bytes()); + server_info.put_i32(len.try_into().unwrap()); + server_info.put_slice(key.as_bytes()); + server_info.put_bytes(0, 1); + server_info.put_slice(value.as_bytes()); + server_info.put_bytes(0, 1); + + return server_info; +} diff --git a/tests/python/tests.py b/tests/python/tests.py index 8eb47f6a4..06d27dced 100644 --- a/tests/python/tests.py +++ b/tests/python/tests.py @@ -1,11 +1,22 @@ import psycopg2 -conn = psycopg2.connect("postgres://random:password@127.0.0.1:6432/db") -cur = conn.cursor() +def test_normal_db_access(): + conn = psycopg2.connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") + cur = conn.cursor() -cur.execute("SELECT 1"); -res = cur.fetchall() + cur.execute("SELECT 1") + res = cur.fetchall() + print(res) -print(res) -# conn.commit() \ No newline at end of file +def test_admin_db_access(): + conn = psycopg2.connect("postgres://user:pass@127.0.0.1:6432/pgcat") + conn.autocommit = True # BEGIN/COMMIT is not supported by admin db + cur = conn.cursor() + + cur.execute("SHOW POOLS") + res = cur.fetchall() + print(res) + +test_normal_db_access() +test_admin_db_access() diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index aaabe5e4e..c5a55a7e3 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -128,3 +128,16 @@ def poorly_behaved_client 25.times do poorly_behaved_client end + + +def test_server_parameters + server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") + raise StandardError, "Bad server version" if server_conn.server_version == 0 + server_conn.close + + admin_conn = PG::connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat") + raise StandardError, "Bad server version" if admin_conn.server_version == 0 + admin_conn.close + + puts 'Server parameters ok' +end From 33e50a5f7423536e475ba8981418768b4891bd30 Mon Sep 17 00:00:00 2001 From: Pradeep Chhetri <30620077+chhetripradeep@users.noreply.github.com> Date: Wed, 3 Aug 2022 00:23:35 +0800 Subject: [PATCH 164/551] Sync pgcat config for docker-compose (#104) --- examples/docker/pgcat.toml | 131 ++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 53 deletions(-) diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index 92c728a79..874f737ab 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -5,21 +5,12 @@ # # General pooler settings [general] - # What IP to run on, 0.0.0.0 means accessible from everywhere. host = "0.0.0.0" # Port to run on, same as PgBouncer used in this example. port = 6432 -# How many connections to allocate per server. -pool_size = 15 - -# Pool mode (see PgBouncer docs for more). -# session: one server connection per connected client -# transaction: one server connection per client transaction -pool_mode = "transaction" - # How long to wait before aborting a server connection (ms). connect_timeout = 5000 @@ -27,52 +18,29 @@ connect_timeout = 5000 healthcheck_timeout = 1000 # For how long to ban a server if it fails a health check (seconds). -ban_time = 60 # Seconds - -# -# User to use for authentication against the server. -[user] -name = "postgres" -password = "postgres" - - -# -# Shards in the cluster -[shards] - -# Shard 0 -[shards.0] - -# [ host, port, role ] -servers = [ - [ "postgres", 5432, "primary" ], - [ "postgres", 5432, "replica" ], - # [ "127.0.1.1", 5432, "replica" ], -] -# Database name (e.g. "postgres") -database = "postgres" +ban_time = 60 # seconds -[shards.1] -# [ host, port, role ] -servers = [ - [ "postgres", 5432, "primary" ], - [ "postgres", 5432, "replica" ], - # [ "127.0.1.1", 5432, "replica" ], -] -database = "postgres" +# Reload config automatically if it changes. +autoreload = false -[shards.2] -# [ host, port, role ] -servers = [ - [ "postgres", 5432, "primary" ], - [ "postgres", 5432, "replica" ], - # [ "127.0.1.1", 5432, "replica" ], -] -database = "postgres" +# TLS +# tls_certificate = "server.cert" +# tls_private_key = "server.key" +# Credentials to access the virtual administrative database (pgbouncer or pgcat) +# Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. +admin_username = "postgres" +admin_password = "postgres" -# Settings for our query routing layer. -[query_router] +# pool +# configs are structured as pool. +# the pool_name is what clients use as database name when connecting +# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded" +[pools.sharded] +# Pool mode (see PgBouncer docs for more). +# session: one server connection per connected client +# transaction: one server connection per client transaction +pool_mode = "transaction" # If the client doesn't specify, route traffic to # this role by default. @@ -82,12 +50,11 @@ database = "postgres" # primary: all queries go to the primary unless otherwise specified. default_role = "any" - # Query parser. If enabled, we'll attempt to parse # every incoming query to determine if it's a read or a write. # If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, # we'll direct it to the primary. -query_parser_enabled = false +query_parser_enabled = true # If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for # load balancing of read queries. Otherwise, the primary will only be used for write @@ -103,3 +70,61 @@ primary_reads_enabled = true # sha1: A hashing function based on SHA1 # sharding_function = "pg_bigint_hash" + +# Credentials for users that may connect to this cluster +[pools.sharded.users.0] +username = "postgres" +password = "postgres" +# Maximum number of server connections that can be established for this user +# The maximum number of connection from a single Pgcat process to any database in the cluster +# is the sum of pool_size across all users. +pool_size = 9 + +[pools.sharded.users.1] +username = "postgres" +password = "postgres" +pool_size = 21 + +# Shard 0 +[pools.sharded.shards.0] +# [ host, port, role ] +servers = [ + [ "postgres", 5432, "primary" ], + [ "postgres", 5432, "replica" ] +] +# Database name (e.g. "postgres") +database = "postgres" + +[pools.sharded.shards.1] +servers = [ + [ "postgres", 5432, "primary" ], + [ "postgres", 5432, "replica" ], +] +database = "postgres" + +[pools.sharded.shards.2] +servers = [ + [ "postgres", 5432, "primary" ], + [ "postgres", 5432, "replica" ], +] +database = "postgres" + + +[pools.simple_db] +pool_mode = "session" +default_role = "primary" +query_parser_enabled = true +primary_reads_enabled = true +sharding_function = "pg_bigint_hash" + +[pools.simple_db.users.0] +username = "postgres" +password = "postgres" +pool_size = 5 + +[pools.simple_db.shards.0] +servers = [ + [ "postgres", 5432, "primary" ], + [ "postgres", 5432, "replica" ] +] +database = "postgres" From 3713276a4c519a51e6f7fe58a63f0b37d6dca422 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 2 Aug 2022 17:52:22 -0500 Subject: [PATCH 165/551] Fix Python tests and remove CircleCI-specific path (#106) * Remove CircleCI-specific path in tests * ..? * Fix testsP * Fix python test * remove pip * Maybe fail? * return code? * no & * Fix tests --- .circleci/config.yml | 2 +- .circleci/run_tests.sh | 18 +++++++++--------- tests/python/tests.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9bb12f06..45ded26a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: command: "cargo fmt --check" - run: name: "Install dependencies" - command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python" + command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip" - run: name: "Build" command: "cargo build" diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index a6fb7909f..138dbecb0 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -66,19 +66,19 @@ psql -U sharding_user -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_te # # ActiveRecord tests # -cd tests/ruby && \ - sudo gem install bundler && \ - bundle install && \ - ruby tests.rb -cd /home/circleci/project +cd tests/ruby +sudo gem install bundler +bundle install +ruby tests.rb +cd ../.. # # Python tests # -cd tests/python && \ - pip install -r requirements.txt && \ - python tests.py -cd /home/circleci/project +cd tests/python +pip3 install -r requirements.txt +python3 tests.py +cd ../.. # Admin tests diff --git a/tests/python/tests.py b/tests/python/tests.py index 06d27dced..15e3822e8 100644 --- a/tests/python/tests.py +++ b/tests/python/tests.py @@ -10,7 +10,7 @@ def test_normal_db_access(): def test_admin_db_access(): - conn = psycopg2.connect("postgres://user:pass@127.0.0.1:6432/pgcat") + conn = psycopg2.connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat") conn.autocommit = True # BEGIN/COMMIT is not supported by admin db cur = conn.cursor() From d9b9c2cfb33515c1367648925d3858878a3f4245 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 3 Aug 2022 20:16:53 -0500 Subject: [PATCH 166/551] Add user to SHOW STATS query (#108) * Add user to SHOW STATS query * user_name => username --- src/admin.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 831ca0b6c..170c851f0 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -370,6 +370,7 @@ where { let columns = vec![ ("database", DataType::Text), + ("user", DataType::Text), ("total_xact_count", DataType::Numeric), ("total_query_count", DataType::Numeric), ("total_received", DataType::Numeric), @@ -390,7 +391,7 @@ where let mut res = BytesMut::new(); res.put(row_description(&columns)); - for (_, pool) in get_all_pools() { + for ((_db_name, username), pool) in get_all_pools() { for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { let address = pool.address(shard, server); @@ -400,8 +401,9 @@ where }; let mut row = vec![address.name()]; + row.push(username.clone()); - for column in &columns[1..] { + for column in &columns[2..] { row.push(stats.get(column.0).unwrap_or(&0).to_string()); } From 037f3f5db8c6494077f1bddbbc03fc4e1ac4cf11 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 8 Aug 2022 09:50:29 -0500 Subject: [PATCH 167/551] Report banned addresses as disabled (#111) --- src/admin.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 170c851f0..5c820ee7f 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -252,10 +252,10 @@ where let pool_config = pool.settings.clone(); for shard in 0..pool.shards() { let database_name = &pool_config.shards[&shard.to_string()].database; - for server in 0..pool.servers(shard) { let address = pool.address(shard, server); let pool_state = pool.pool_state(shard, server); + let banned = pool.is_banned(address, shard, Some(address.role)); res.put(data_row(&vec![ address.name(), // name @@ -270,7 +270,11 @@ where pool_config.user.pool_size.to_string(), // max_connections pool_state.connections.to_string(), // current_connections "0".to_string(), // paused - "0".to_string(), // disabled + match banned { + // disabled + true => "1".to_string(), + false => "0".to_string(), + }, ])); } } From ec5c255d84df322f5dd7987678bfe3fa306bc248 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 8 Aug 2022 09:51:36 -0500 Subject: [PATCH 168/551] Generate test coverage report in CircleCI (#110) * coverage? * generate_coverage * +x * 1.62.1 * 62 * ignore * store * quote --- .circleci/config.yml | 19 ++++++++++++------- .circleci/generate_coverage.sh | 7 +++++++ 2 files changed, 19 insertions(+), 7 deletions(-) create mode 100755 .circleci/generate_coverage.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 45ded26a2..aed5c182d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,9 +9,11 @@ jobs: # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - - image: cimg/rust:1.58.1 + - image: cimg/rust:1.62.0 environment: RUST_LOG: info + RUSTFLAGS: "-C instrument-coverage" + LLVM_PROFILE_FILE: "pgcat-%m.profraw" - image: postgres:14 # auth: # username: mydockerhub-user @@ -32,16 +34,19 @@ jobs: command: "cargo fmt --check" - run: name: "Install dependencies" - command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip" + command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip lcov llvm-11" + - run: + name: "Install rust tools" + command: "cargo install cargo-binutils rustfilt && rustup component add llvm-tools-preview" - run: name: "Build" command: "cargo build" - run: - name: "Test" - command: "cargo test" - - run: - name: "Test end-to-end" - command: "bash .circleci/run_tests.sh" + name: "Tests" + command: "cargo test && bash .circleci/run_tests.sh && .circleci/generate_coverage.sh" + - store_artifacts: + path: /tmp/cov + destination: coverage-data - save_cache: key: cargo-lock-2-{{ checksum "Cargo.lock" }} paths: diff --git a/.circleci/generate_coverage.sh b/.circleci/generate_coverage.sh new file mode 100755 index 000000000..59fe44aa8 --- /dev/null +++ b/.circleci/generate_coverage.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +rust-profdata merge -sparse pgcat-*.profraw -o pgcat.profdata + +rust-cov export -ignore-filename-regex="rustc|registry" -Xdemangler=rustfilt -instr-profile=pgcat.profdata --object ./target/debug/pgcat --format lcov > ./lcov.info + +genhtml lcov.info --output-directory /tmp/cov --prefix $(pwd) From 9cfffaa27ff762cbd839badcd5d5bb9b97d322b2 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 8 Aug 2022 15:15:48 -0500 Subject: [PATCH 169/551] Fix local dev (#112) * Fix Dev env * Update tests/sharding/query_routing_setup.sql * Update tests/sharding/query_routing_setup.sql * bring pgcat.toml on ci and local dev to parity * more parity * pool names * pool names * less diff * fix tests * fmt * add other user to setup --- .circleci/pgcat.toml | 27 +++++++++++++++++- pgcat.toml | 20 ++++++------- src/config.rs | 21 ++++++++------ tests/sharding/query_routing_setup.sql | 39 +++++++++++++++++++++++--- 4 files changed, 83 insertions(+), 24 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 1c0c01040..7afb42012 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -35,7 +35,7 @@ admin_password = "admin_pass" # pool # configs are structured as pool. # the pool_name is what clients use as database name when connecting -# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded" +# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded_db" [pools.sharded_db] # Pool mode (see PgBouncer docs for more). # session: one server connection per connected client @@ -80,6 +80,11 @@ password = "sharding_user" # is the sum of pool_size across all users. pool_size = 9 +[pools.sharded_db.users.1] +username = "other_user" +password = "other_user" +pool_size = 21 + # Shard 0 [pools.sharded_db.shards.0] # [ host, port, role ] @@ -103,3 +108,23 @@ servers = [ [ "localhost", 5432, "replica" ], ] database = "shard2" + + +[pools.simple_db] +pool_mode = "session" +default_role = "primary" +query_parser_enabled = true +primary_reads_enabled = true +sharding_function = "pg_bigint_hash" + +[pools.simple_db.users.0] +username = "simple_user" +password = "simple_user" +pool_size = 5 + +[pools.simple_db.shards.0] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ] +] +database = "some_db" diff --git a/pgcat.toml b/pgcat.toml index a1937e6cb..0fa53bcdc 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -14,7 +14,7 @@ port = 6432 # How long to wait before aborting a server connection (ms). connect_timeout = 5000 -# How much time to give `SELECT 1` health check query to return with a result (ms). +# How much time to give the health check query to return with a result (ms). healthcheck_timeout = 1000 # For how long to ban a server if it fails a health check (seconds). @@ -29,14 +29,14 @@ autoreload = false # Credentials to access the virtual administrative database (pgbouncer or pgcat) # Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. -admin_username = "user" -admin_password = "pass" +admin_username = "admin_user" +admin_password = "admin_pass" # pool # configs are structured as pool. # the pool_name is what clients use as database name when connecting -# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded" -[pools.sharded] +# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded_db" +[pools.sharded_db] # Pool mode (see PgBouncer docs for more). # session: one server connection per connected client # transaction: one server connection per client transaction @@ -72,7 +72,7 @@ primary_reads_enabled = true sharding_function = "pg_bigint_hash" # Credentials for users that may connect to this cluster -[pools.sharded.users.0] +[pools.sharded_db.users.0] username = "sharding_user" password = "sharding_user" # Maximum number of server connections that can be established for this user @@ -80,13 +80,13 @@ password = "sharding_user" # is the sum of pool_size across all users. pool_size = 9 -[pools.sharded.users.1] +[pools.sharded_db.users.1] username = "other_user" password = "other_user" pool_size = 21 # Shard 0 -[pools.sharded.shards.0] +[pools.sharded_db.shards.0] # [ host, port, role ] servers = [ [ "127.0.0.1", 5432, "primary" ], @@ -95,14 +95,14 @@ servers = [ # Database name (e.g. "postgres") database = "shard0" -[pools.sharded.shards.1] +[pools.sharded_db.shards.1] servers = [ [ "127.0.0.1", 5432, "primary" ], [ "localhost", 5432, "replica" ], ] database = "shard1" -[pools.sharded.shards.2] +[pools.sharded_db.shards.2] servers = [ [ "127.0.0.1", 5432, "primary" ], [ "localhost", 5432, "replica" ], diff --git a/src/config.rs b/src/config.rs index f1138f98b..8e5e94e9e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -521,30 +521,33 @@ mod test { assert_eq!(get_config().general.ban_time, 60); assert_eq!(get_config().pools.len(), 2); - assert_eq!(get_config().pools["sharded"].shards.len(), 3); + assert_eq!(get_config().pools["sharded_db"].shards.len(), 3); assert_eq!(get_config().pools["simple_db"].shards.len(), 1); - assert_eq!(get_config().pools["sharded"].users.len(), 2); + assert_eq!(get_config().pools["sharded_db"].users.len(), 2); assert_eq!(get_config().pools["simple_db"].users.len(), 1); assert_eq!( - get_config().pools["sharded"].shards["0"].servers[0].0, + get_config().pools["sharded_db"].shards["0"].servers[0].0, "127.0.0.1" ); assert_eq!( - get_config().pools["sharded"].shards["1"].servers[0].2, + get_config().pools["sharded_db"].shards["1"].servers[0].2, "primary" ); - assert_eq!(get_config().pools["sharded"].shards["1"].database, "shard1"); assert_eq!( - get_config().pools["sharded"].users["0"].username, + get_config().pools["sharded_db"].shards["1"].database, + "shard1" + ); + assert_eq!( + get_config().pools["sharded_db"].users["0"].username, "sharding_user" ); assert_eq!( - get_config().pools["sharded"].users["1"].password, + get_config().pools["sharded_db"].users["1"].password, "other_user" ); - assert_eq!(get_config().pools["sharded"].users["1"].pool_size, 21); - assert_eq!(get_config().pools["sharded"].default_role, "any"); + assert_eq!(get_config().pools["sharded_db"].users["1"].pool_size, 21); + assert_eq!(get_config().pools["sharded_db"].default_role, "any"); assert_eq!( get_config().pools["simple_db"].shards["0"].servers[0].0, diff --git a/tests/sharding/query_routing_setup.sql b/tests/sharding/query_routing_setup.sql index 8ecfc58a8..384b23414 100644 --- a/tests/sharding/query_routing_setup.sql +++ b/tests/sharding/query_routing_setup.sql @@ -1,11 +1,12 @@ - DROP DATABASE IF EXISTS shard0; DROP DATABASE IF EXISTS shard1; DROP DATABASE IF EXISTS shard2; +DROP DATABASE IF EXISTS some_db; CREATE DATABASE shard0; CREATE DATABASE shard1; CREATE DATABASE shard2; +CREATE DATABASE some_db; \c shard0 @@ -41,21 +42,51 @@ CREATE TABLE data ( CREATE TABLE data_shard_2 PARTITION OF data FOR VALUES WITH (MODULUS 3, REMAINDER 2); + +\c some_db + +DROP TABLE IF EXISTS data CASCADE; + +CREATE TABLE data ( + id BIGINT, + value VARCHAR +); + DROP ROLE IF EXISTS sharding_user; +DROP ROLE IF EXISTS other_user; +DROP ROLE IF EXISTS simple_user; CREATE ROLE sharding_user ENCRYPTED PASSWORD 'sharding_user' LOGIN; +CREATE ROLE other_user ENCRYPTED PASSWORD 'other_user' LOGIN; +CREATE ROLE simple_user ENCRYPTED PASSWORD 'simple_user' LOGIN; -GRANT CONNECT ON DATABASE shard0 TO sharding_user; -GRANT CONNECT ON DATABASE shard1 TO sharding_user; -GRANT CONNECT ON DATABASE shard2 TO sharding_user; +GRANT CONNECT ON DATABASE shard0 TO sharding_user; +GRANT CONNECT ON DATABASE shard1 TO sharding_user; +GRANT CONNECT ON DATABASE shard2 TO sharding_user; + +GRANT CONNECT ON DATABASE shard0 TO other_user; +GRANT CONNECT ON DATABASE shard1 TO other_user; +GRANT CONNECT ON DATABASE shard2 TO other_user; + +GRANT CONNECT ON DATABASE some_db TO simple_user; \c shard0 GRANT ALL ON SCHEMA public TO sharding_user; GRANT ALL ON TABLE data TO sharding_user; +GRANT ALL ON SCHEMA public TO other_user; +GRANT ALL ON TABLE data TO other_user; \c shard1 GRANT ALL ON SCHEMA public TO sharding_user; GRANT ALL ON TABLE data TO sharding_user; +GRANT ALL ON SCHEMA public TO other_user; +GRANT ALL ON TABLE data TO other_user; \c shard2 GRANT ALL ON SCHEMA public TO sharding_user; GRANT ALL ON TABLE data TO sharding_user; +GRANT ALL ON SCHEMA public TO other_user; +GRANT ALL ON TABLE data TO other_user; + +\c some_db +GRANT ALL ON SCHEMA public TO simple_user; +GRANT ALL ON TABLE data TO simple_user; From 1bfbbea300d1e859980d0b393bc8248c1533c833 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Mon, 8 Aug 2022 19:01:24 -0400 Subject: [PATCH 170/551] Implementing graceful shutdown (#105) * Initial commit for graceful shutdown * fmt * Add .vscode to gitignore * Updates shutdown logic to use channels * fmt * fmt * Adds shutdown timeout * Fmt and updates tomls * Updates readme * fmt and updates log levels * Update python tests to test shutdown * merge changes * Rename listener rx and update bash to be in line with master * Update python test bash script ordering * Adds error response message before shutdown * Add details on shutdown event loop * Fixes response length for error * Adds handler for sigterm * Uses ready for query function and fixes number of bytes * fmt --- .circleci/pgcat.toml | 3 + .circleci/run_tests.sh | 8 +- .gitignore | 1 + README.md | 2 + examples/docker/pgcat.toml | 3 + pgcat.toml | 3 + src/client.rs | 63 +++++++++++++-- src/config.rs | 7 ++ src/main.rs | 78 +++++++++++++++--- src/messages.rs | 42 +++++----- tests/python/requirements.txt | 1 + tests/python/tests.py | 144 +++++++++++++++++++++++++++++++++- 12 files changed, 308 insertions(+), 47 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 7afb42012..502215b06 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -17,6 +17,9 @@ connect_timeout = 100 # How much time to give the health check query to return with a result (ms). healthcheck_timeout = 100 +# How much time to give clients during shutdown before forcibly killing client connections (ms). +shutdown_timeout = 5000 + # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # Seconds diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 138dbecb0..431f2d604 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -74,12 +74,12 @@ cd ../.. # # Python tests +# These tests will start and stop the pgcat server so it will need to be restarted after the tests # -cd tests/python -pip3 install -r requirements.txt -python3 tests.py -cd ../.. +pip3 install -r tests/python/requirements.txt +python3 tests/python/tests.py +start_pgcat "info" # Admin tests export PGPASSWORD=admin_pass diff --git a/.gitignore b/.gitignore index a4b784117..3c6545390 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea /target *.deb +.vscode \ No newline at end of file diff --git a/README.md b/README.md index 5b54a7fc3..c03982d2b 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ psql -h 127.0.0.1 -p 6432 -c 'SELECT 1' | `pool_mode` | The pool mode to use, i.e. `session` or `transaction`. | `transaction` | | `connect_timeout` | Maximum time to establish a connection to a server (milliseconds). If reached, the server is banned and the next target is attempted. | `5000` | | `healthcheck_timeout` | Maximum time to pass a health check (`SELECT 1`, milliseconds). If reached, the server is banned and the next target is attempted. | `1000` | +| `shutdown_timeout` | Maximum time to give clients during shutdown before forcibly killing client connections (ms). | `60000` | | `ban_time` | Ban time for a server (seconds). It won't be allowed to serve transactions until the ban expires; failover targets will be used instead. | `60` | | | | | | **`user`** | | | @@ -250,6 +251,7 @@ The config can be reloaded by sending a `kill -s SIGHUP` to the process or by qu | `pool_mode` | no | | `connect_timeout` | yes | | `healthcheck_timeout` | no | +| `shutdown_timeout` | no | | `ban_time` | no | | `user` | yes | | `shards` | yes | diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index 874f737ab..40a54928c 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -17,6 +17,9 @@ connect_timeout = 5000 # How much time to give `SELECT 1` health check query to return with a result (ms). healthcheck_timeout = 1000 +# How much time to give clients during shutdown before forcibly killing client connections (ms). +shutdown_timeout = 60000 + # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # seconds diff --git a/pgcat.toml b/pgcat.toml index 0fa53bcdc..50d797f81 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -17,6 +17,9 @@ connect_timeout = 5000 # How much time to give the health check query to return with a result (ms). healthcheck_timeout = 1000 +# How much time to give clients during shutdown before forcibly killing client connections (ms). +shutdown_timeout = 60000 + # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # seconds diff --git a/src/client.rs b/src/client.rs index 1775ad22a..cc9121915 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,6 +4,7 @@ use log::{debug, error, info, trace}; use std::collections::HashMap; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; use tokio::net::TcpStream; +use tokio::sync::broadcast::Receiver; use crate::admin::{generate_server_info_for_admin, handle_admin}; use crate::config::get_config; @@ -73,12 +74,15 @@ pub struct Client { last_server_id: Option, target_pool: ConnectionPool, + + shutdown_event_receiver: Receiver<()>, } /// Client entrypoint. pub async fn client_entrypoint( mut stream: TcpStream, client_server_map: ClientServerMap, + shutdown_event_receiver: Receiver<()>, ) -> Result<(), Error> { // Figure out if the client wants TLS or not. let addr = stream.peer_addr().unwrap(); @@ -97,7 +101,7 @@ pub async fn client_entrypoint( write_all(&mut stream, yes).await?; // Negotiate TLS. - match startup_tls(stream, client_server_map).await { + match startup_tls(stream, client_server_map, shutdown_event_receiver).await { Ok(mut client) => { info!("Client {:?} connected (TLS)", addr); @@ -121,7 +125,16 @@ pub async fn client_entrypoint( let (read, write) = split(stream); // Continue with regular startup. - match Client::startup(read, write, addr, bytes, client_server_map).await { + match Client::startup( + read, + write, + addr, + bytes, + client_server_map, + shutdown_event_receiver, + ) + .await + { Ok(mut client) => { info!("Client {:?} connected (plain)", addr); @@ -142,7 +155,16 @@ pub async fn client_entrypoint( let (read, write) = split(stream); // Continue with regular startup. - match Client::startup(read, write, addr, bytes, client_server_map).await { + match Client::startup( + read, + write, + addr, + bytes, + client_server_map, + shutdown_event_receiver, + ) + .await + { Ok(mut client) => { info!("Client {:?} connected (plain)", addr); @@ -157,7 +179,16 @@ pub async fn client_entrypoint( let (read, write) = split(stream); // Continue with cancel query request. - match Client::cancel(read, write, addr, bytes, client_server_map).await { + match Client::cancel( + read, + write, + addr, + bytes, + client_server_map, + shutdown_event_receiver, + ) + .await + { Ok(mut client) => { info!("Client {:?} issued a cancel query request", addr); @@ -214,6 +245,7 @@ where pub async fn startup_tls( stream: TcpStream, client_server_map: ClientServerMap, + shutdown_event_receiver: Receiver<()>, ) -> Result>, WriteHalf>>, Error> { // Negotiate TLS. let tls = Tls::new()?; @@ -237,7 +269,15 @@ pub async fn startup_tls( Ok((ClientConnectionType::Startup, bytes)) => { let (read, write) = split(stream); - Client::startup(read, write, addr, bytes, client_server_map).await + Client::startup( + read, + write, + addr, + bytes, + client_server_map, + shutdown_event_receiver, + ) + .await } // Bad Postgres client. @@ -258,6 +298,7 @@ where addr: std::net::SocketAddr, bytes: BytesMut, // The rest of the startup message. client_server_map: ClientServerMap, + shutdown_event_receiver: Receiver<()>, ) -> Result, Error> { let config = get_config(); let stats = get_reporter(); @@ -384,6 +425,7 @@ where last_address_id: None, last_server_id: None, target_pool: target_pool, + shutdown_event_receiver: shutdown_event_receiver, }); } @@ -394,6 +436,7 @@ where addr: std::net::SocketAddr, mut bytes: BytesMut, // The rest of the startup message. client_server_map: ClientServerMap, + shutdown_event_receiver: Receiver<()>, ) -> Result, Error> { let process_id = bytes.get_i32(); let secret_key = bytes.get_i32(); @@ -413,6 +456,7 @@ where last_address_id: None, last_server_id: None, target_pool: ConnectionPool::default(), + shutdown_event_receiver: shutdown_event_receiver, }); } @@ -467,7 +511,14 @@ where // We can parse it here before grabbing a server from the pool, // in case the client is sending some custom protocol messages, e.g. // SET SHARDING KEY TO 'bigint'; - let mut message = read_message(&mut self.read).await?; + + let mut message = tokio::select! { + _ = self.shutdown_event_receiver.recv() => { + error_response_terminal(&mut self.write, &format!("terminating connection due to administrator command")).await?; + return Ok(()) + }, + message_result = read_message(&mut self.read) => message_result? + }; // Get a pool instance referenced by the most up-to-date // pointer. This ensures we always read the latest config diff --git a/src/config.rs b/src/config.rs index 8e5e94e9e..05a1b3672 100644 --- a/src/config.rs +++ b/src/config.rs @@ -119,6 +119,7 @@ pub struct General { pub port: i16, pub connect_timeout: u64, pub healthcheck_timeout: u64, + pub shutdown_timeout: u64, pub ban_time: i64, pub autoreload: bool, pub tls_certificate: Option, @@ -134,6 +135,7 @@ impl Default for General { port: 5432, connect_timeout: 5000, healthcheck_timeout: 1000, + shutdown_timeout: 60000, ban_time: 60, autoreload: false, tls_certificate: None, @@ -273,6 +275,10 @@ impl From<&Config> for std::collections::HashMap { "healthcheck_timeout".to_string(), config.general.healthcheck_timeout.to_string(), ), + ( + "shutdown_timeout".to_string(), + config.general.shutdown_timeout.to_string(), + ), ("ban_time".to_string(), config.general.ban_time.to_string()), ]; @@ -290,6 +296,7 @@ impl Config { self.general.healthcheck_timeout ); info!("Connection timeout: {}ms", self.general.connect_timeout); + info!("Shutdown timeout: {}ms", self.general.shutdown_timeout); match self.general.tls_certificate.clone() { Some(tls_certificate) => { info!("TLS certificate: {}", tls_certificate); diff --git a/src/main.rs b/src/main.rs index 3622398c3..5e5c9248e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -40,13 +40,13 @@ use log::{debug, error, info}; use parking_lot::Mutex; use tokio::net::TcpListener; use tokio::{ - signal, signal::unix::{signal as unix_signal, SignalKind}, sync::mpsc, }; use std::collections::HashMap; use std::sync::Arc; +use tokio::sync::broadcast; mod admin; mod client; @@ -139,24 +139,52 @@ async fn main() { info!("Waiting for clients"); + let (shutdown_event_tx, mut shutdown_event_rx) = broadcast::channel::<()>(1); + + let shutdown_event_tx_clone = shutdown_event_tx.clone(); + // Client connection loop. tokio::task::spawn(async move { + // Creates event subscriber for shutdown event, this is dropped when shutdown event is broadcast + let mut listener_shutdown_event_rx = shutdown_event_tx_clone.subscribe(); loop { let client_server_map = client_server_map.clone(); - let (socket, addr) = match listener.accept().await { - Ok((socket, addr)) => (socket, addr), - Err(err) => { - error!("{:?}", err); - continue; + // Listen for shutdown event and client connection at the same time + let (socket, addr) = tokio::select! { + _ = listener_shutdown_event_rx.recv() => { + // Exits client connection loop which drops listener, listener_shutdown_event_rx and shutdown_event_tx_clone + break; + } + + listener_response = listener.accept() => { + match listener_response { + Ok((socket, addr)) => (socket, addr), + Err(err) => { + error!("{:?}", err); + continue; + } + } } }; + // Used to signal shutdown + let client_shutdown_handler_rx = shutdown_event_tx_clone.subscribe(); + + // Used to signal that the task has completed + let dummy_tx = shutdown_event_tx_clone.clone(); + // Handle client. tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); - match client::client_entrypoint(socket, client_server_map).await { + match client::client_entrypoint( + socket, + client_server_map, + client_shutdown_handler_rx, + ) + .await + { Ok(_) => { let duration = chrono::offset::Utc::now().naive_utc() - start; @@ -171,6 +199,8 @@ async fn main() { debug!("Client disconnected with error {:?}", err); } }; + // Drop this transmitter so receiver knows that the task is completed + drop(dummy_tx); }); } }); @@ -214,13 +244,41 @@ async fn main() { }); } - // Exit on Ctrl-C (SIGINT) and SIGTERM. let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); + let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); tokio::select! { - _ = signal::ctrl_c() => (), + // Initiate graceful shutdown sequence on sig int + _ = interrupt_signal.recv() => { + info!("Got SIGINT, waiting for client connection drain now"); + + // Broadcast that client tasks need to finish + shutdown_event_tx.send(()).unwrap(); + // Closes transmitter + drop(shutdown_event_tx); + + // This is in a loop because the first event that the receiver receives will be the shutdown event + // This is not what we are waiting for instead, we want the receiver to send an error once all senders are closed which is reached after the shutdown event is received + loop { + match tokio::time::timeout( + tokio::time::Duration::from_millis(config.general.shutdown_timeout), + shutdown_event_rx.recv(), + ) + .await + { + Ok(res) => match res { + Ok(_) => {} + Err(_) => break, + }, + Err(_) => { + info!("Timed out while waiting for clients to shutdown"); + break; + } + } + } + }, _ = term_signal.recv() => (), - }; + } info!("Shutting down..."); } diff --git a/src/messages.rs b/src/messages.rs index ba22a5791..113e1ed52 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -98,7 +98,9 @@ pub async fn ready_for_query(stream: &mut S) -> Result<(), Error> where S: tokio::io::AsyncWrite + std::marker::Unpin, { - let mut bytes = BytesMut::with_capacity(5); + let mut bytes = BytesMut::with_capacity( + mem::size_of::() + mem::size_of::() + mem::size_of::(), + ); bytes.put_u8(b'Z'); bytes.put_i32(5); @@ -252,18 +254,25 @@ where res.put_i32(len); res.put_slice(&set_complete[..]); - // ReadyForQuery (idle) - res.put_u8(b'Z'); - res.put_i32(5); - res.put_u8(b'I'); - - write_all_half(stream, res).await + write_all_half(stream, res).await?; + ready_for_query(stream).await } /// Send a custom error message to the client. /// Tell the client we are ready for the next query and no rollback is necessary. /// Docs on error codes: . pub async fn error_response(stream: &mut S, message: &str) -> Result<(), Error> +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ + error_response_terminal(stream, message).await?; + ready_for_query(stream).await +} + +/// Send a custom error message to the client. +/// Tell the client we are ready for the next query and no rollback is necessary. +/// Docs on error codes: . +pub async fn error_response_terminal(stream: &mut S, message: &str) -> Result<(), Error> where S: tokio::io::AsyncWrite + std::marker::Unpin, { @@ -288,21 +297,12 @@ where // No more fields follow. error.put_u8(0); - // Ready for query, no rollback needed (I = idle). - let mut ready_for_query = BytesMut::new(); - - ready_for_query.put_u8(b'Z'); - ready_for_query.put_i32(5); - ready_for_query.put_u8(b'I'); - // Compose the two message reply. - let mut res = BytesMut::with_capacity(error.len() + ready_for_query.len() + 5); + let mut res = BytesMut::with_capacity(error.len() + 5); res.put_u8(b'E'); res.put_i32(error.len() as i32 + 4); - res.put(error); - res.put(ready_for_query); Ok(write_all_half(stream, res).await?) } @@ -366,12 +366,8 @@ where // CommandComplete res.put(command_complete("SELECT 1")); - // ReadyForQuery - res.put_u8(b'Z'); - res.put_i32(5); - res.put_u8(b'I'); - - write_all_half(stream, res).await + write_all_half(stream, res).await?; + ready_for_query(stream).await } pub fn row_description(columns: &Vec<(&str, DataType)>) -> BytesMut { diff --git a/tests/python/requirements.txt b/tests/python/requirements.txt index d7661d4d9..eebd9c90a 100644 --- a/tests/python/requirements.txt +++ b/tests/python/requirements.txt @@ -1 +1,2 @@ psycopg2==2.9.3 +psutil==5.9.1 \ No newline at end of file diff --git a/tests/python/tests.py b/tests/python/tests.py index 15e3822e8..3ff99a091 100644 --- a/tests/python/tests.py +++ b/tests/python/tests.py @@ -1,22 +1,158 @@ +from typing import Tuple import psycopg2 +import psutil +import os +import signal +import subprocess +from threading import Thread +import time -def test_normal_db_access(): - conn = psycopg2.connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") +SHUTDOWN_TIMEOUT = 5 + +PGCAT_HOST = "127.0.0.1" +PGCAT_PORT = "6432" + + +def pgcat_start(): + pg_cat_send_signal(signal.SIGTERM) + pgcat_start_command = "./target/debug/pgcat .circleci/pgcat.toml" + subprocess.Popen(pgcat_start_command.split()) + + +def pg_cat_send_signal(signal: signal.Signals): + for proc in psutil.process_iter(["pid", "name"]): + if "pgcat" == proc.name(): + os.kill(proc.pid, signal) + + +def connect_normal_db( + autocommit: bool = False, +) -> Tuple[psycopg2.extensions.connection, psycopg2.extensions.cursor]: + conn = psycopg2.connect( + f"postgres://sharding_user:sharding_user@{PGCAT_HOST}:{PGCAT_PORT}/sharded_db?application_name=testing_pgcat" + ) + conn.autocommit = autocommit cur = conn.cursor() + return (conn, cur) + + +def cleanup_conn(conn: psycopg2.extensions.connection, cur: psycopg2.extensions.cursor): + cur.close() + conn.close() + + +def test_normal_db_access(): + conn, cur = connect_normal_db() cur.execute("SELECT 1") res = cur.fetchall() print(res) + cleanup_conn(conn, cur) def test_admin_db_access(): - conn = psycopg2.connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat") - conn.autocommit = True # BEGIN/COMMIT is not supported by admin db + conn = psycopg2.connect( + f"postgres://admin_user:admin_pass@{PGCAT_HOST}:{PGCAT_PORT}/pgcat" + ) + conn.autocommit = True # BEGIN/COMMIT is not supported by admin db cur = conn.cursor() cur.execute("SHOW POOLS") res = cur.fetchall() print(res) + cleanup_conn(conn, cur) + + +def test_shutdown_logic(): + + ##### NO ACTIVE QUERIES SIGINT HANDLING ##### + # Start pgcat + server = Thread(target=pgcat_start) + server.start() + + # Wait for server to fully start up + time.sleep(2) + + # Create client connection and send query (not in transaction) + conn, cur = connect_normal_db(True) + + cur.execute("BEGIN;") + cur.execute("SELECT 1;") + cur.execute("COMMIT;") + + # Send sigint to pgcat + pg_cat_send_signal(signal.SIGINT) + time.sleep(1) + + # Check that any new queries fail after sigint since server should close with no active transactions + try: + cur.execute("SELECT 1;") + except psycopg2.OperationalError as e: + pass + else: + # Fail if query execution succeeded + raise Exception("Server not closed after sigint") + cleanup_conn(conn, cur) + + ##### HANDLE TRANSACTION WITH SIGINT ##### + # Start pgcat + server = Thread(target=pgcat_start) + server.start() + + # Wait for server to fully start up + time.sleep(2) + + # Create client connection and begin transaction + conn, cur = connect_normal_db(True) + + cur.execute("BEGIN;") + cur.execute("SELECT 1;") + + # Send sigint to pgcat while still in transaction + pg_cat_send_signal(signal.SIGINT) + time.sleep(1) + + # Check that any new queries succeed after sigint since server should still allow transaction to complete + try: + cur.execute("SELECT 1;") + except psycopg2.OperationalError as e: + # Fail if query fails since server closed + raise Exception("Server closed while in transaction", e.pgerror) + + cleanup_conn(conn, cur) + + ##### HANDLE SHUTDOWN TIMEOUT WITH SIGINT ##### + # Start pgcat + server = Thread(target=pgcat_start) + server.start() + + # Wait for server to fully start up + time.sleep(3) + + # Create client connection and begin transaction, which should prevent server shutdown unless shutdown timeout is reached + conn, cur = connect_normal_db(True) + + cur.execute("BEGIN;") + cur.execute("SELECT 1;") + + # Send sigint to pgcat while still in transaction + pg_cat_send_signal(signal.SIGINT) + + # pgcat shutdown timeout is set to SHUTDOWN_TIMEOUT seconds, so we sleep for SHUTDOWN_TIMEOUT + 1 seconds + time.sleep(SHUTDOWN_TIMEOUT + 1) + + # Check that any new queries succeed after sigint since server should still allow transaction to complete + try: + cur.execute("SELECT 1;") + except psycopg2.OperationalError as e: + pass + else: + # Fail if query execution succeeded + raise Exception("Server not closed after sigint and expected timeout") + + cleanup_conn(conn, cur) + test_normal_db_access() test_admin_db_access() +test_shutdown_logic() From e37f20c70962e9eeb2f601d5b2e814a7a9eac0e1 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 9 Aug 2022 14:18:27 -0500 Subject: [PATCH 171/551] Prevent clients from sticking to old pools after config update (#113) * Re-acquire pool at the beginning of Protocol loop * Fix query router + add tests for recycling behavior --- src/client.rs | 68 ++++++++++++++++++++++++--------------- src/query_router.rs | 69 +++++++++++++++++++++++++++++++++------- tests/ruby/.ruby-version | 3 +- tests/ruby/Gemfile | 1 + tests/ruby/Gemfile.lock | 5 +++ tests/ruby/tests.rb | 60 ++++++++++++++++++++++++++++++++++ 6 files changed, 168 insertions(+), 38 deletions(-) diff --git a/src/client.rs b/src/client.rs index cc9121915..9d4f40389 100644 --- a/src/client.rs +++ b/src/client.rs @@ -11,7 +11,7 @@ use crate::config::get_config; use crate::constants::*; use crate::errors::Error; use crate::messages::*; -use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; +use crate::pool::{get_pool, ClientServerMap}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; use crate::stats::{get_reporter, Reporter}; @@ -73,8 +73,13 @@ pub struct Client { /// Last server process id we talked to. last_server_id: Option, - target_pool: ConnectionPool, + /// Name of the server pool for this client (This comes from the database name in the connection string) + target_pool_name: String, + /// Postgres user for this client (This comes from the user in the connection string) + target_user_name: String, + + /// Used to notify clients about an impending shutdown shutdown_event_receiver: Receiver<()>, } @@ -305,19 +310,19 @@ where trace!("Got StartupMessage"); let parameters = parse_startup(bytes.clone())?; - let database = match parameters.get("database") { + let target_pool_name = match parameters.get("database") { Some(db) => db, None => return Err(Error::ClientError), }; - let user = match parameters.get("user") { + let target_user_name = match parameters.get("user") { Some(user) => user, None => return Err(Error::ClientError), }; let admin = ["pgcat", "pgbouncer"] .iter() - .filter(|db| *db == &database) + .filter(|db| *db == &target_pool_name) .count() == 1; @@ -352,7 +357,7 @@ where Err(_) => return Err(Error::SocketError), }; - let (target_pool, transaction_mode, server_info) = if admin { + let (transaction_mode, server_info) = if admin { let correct_user = config.general.admin_username.as_str(); let correct_password = config.general.admin_password.as_str(); @@ -360,23 +365,20 @@ where let password_hash = md5_hash_password(correct_user, correct_password, &salt); if password_hash != password_response { debug!("Password authentication failed"); - wrong_password(&mut write, user).await?; + wrong_password(&mut write, target_user_name).await?; return Err(Error::ClientError); } - ( - ConnectionPool::default(), - false, - generate_server_info_for_admin(), - ) + + (false, generate_server_info_for_admin()) } else { - let target_pool = match get_pool(database.clone(), user.clone()) { + let target_pool = match get_pool(target_pool_name.clone(), target_user_name.clone()) { Some(pool) => pool, None => { error_response( &mut write, &format!( "No pool configured for database: {:?}, user: {:?}", - database, user + target_pool_name, target_user_name ), ) .await?; @@ -387,14 +389,14 @@ where let server_info = target_pool.server_info(); // Compare server and client hashes. let correct_password = target_pool.settings.user.password.as_str(); - let password_hash = md5_hash_password(user, correct_password, &salt); + let password_hash = md5_hash_password(&target_user_name, correct_password, &salt); if password_hash != password_response { debug!("Password authentication failed"); - wrong_password(&mut write, user).await?; + wrong_password(&mut write, &target_user_name).await?; return Err(Error::ClientError); } - (target_pool, transaction_mode, server_info) + (transaction_mode, server_info) }; debug!("Password authentication successful"); @@ -424,7 +426,8 @@ where admin: admin, last_address_id: None, last_server_id: None, - target_pool: target_pool, + target_pool_name: target_pool_name.clone(), + target_user_name: target_user_name.clone(), shutdown_event_receiver: shutdown_event_receiver, }); } @@ -455,7 +458,8 @@ where admin: false, last_address_id: None, last_server_id: None, - target_pool: ConnectionPool::default(), + target_pool_name: String::from("undefined"), + target_user_name: String::from("undefined"), shutdown_event_receiver: shutdown_event_receiver, }); } @@ -494,7 +498,7 @@ where // The query router determines where the query is going to go, // e.g. primary, replica, which shard. - let mut query_router = QueryRouter::new(self.target_pool.clone()); + let mut query_router = QueryRouter::new(); let mut round_robin = 0; // Our custom protocol loop. @@ -520,11 +524,6 @@ where message_result = read_message(&mut self.read) => message_result? }; - // Get a pool instance referenced by the most up-to-date - // pointer. This ensures we always read the latest config - // when starting a query. - let mut pool = self.target_pool.clone(); - // Avoid taking a server if the client just wants to disconnect. if message[0] as char == 'X' { debug!("Client disconnecting"); @@ -538,6 +537,25 @@ where continue; } + // Get a pool instance referenced by the most up-to-date + // pointer. This ensures we always read the latest config + // when starting a query. + let mut pool = + match get_pool(self.target_pool_name.clone(), self.target_user_name.clone()) { + Some(pool) => pool, + None => { + error_response( + &mut self.write, + &format!( + "No pool configured for database: {:?}, user: {:?}", + self.target_pool_name, self.target_user_name + ), + ) + .await?; + return Err(Error::ClientError); + } + }; + query_router.update_pool_settings(pool.settings.clone()); let current_shard = query_router.shard(); // Handle all custom protocol commands, if any. diff --git a/src/query_router.rs b/src/query_router.rs index d597b81ee..6b377684f 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -9,7 +9,7 @@ use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; use crate::config::Role; -use crate::pool::{ConnectionPool, PoolSettings}; +use crate::pool::PoolSettings; use crate::sharding::{Sharder, ShardingFunction}; /// Regexes used to parse custom commands. @@ -91,16 +91,20 @@ impl QueryRouter { } /// Create a new instance of the query router. Each client gets its own. - pub fn new(target_pool: ConnectionPool) -> QueryRouter { + pub fn new() -> QueryRouter { QueryRouter { active_shard: None, active_role: None, - query_parser_enabled: target_pool.settings.query_parser_enabled, - primary_reads_enabled: target_pool.settings.primary_reads_enabled, - pool_settings: target_pool.settings, + query_parser_enabled: false, + primary_reads_enabled: false, + pool_settings: PoolSettings::default(), } } + pub fn update_pool_settings(&mut self, pool_settings: PoolSettings) { + self.pool_settings = pool_settings; + } + /// Try to parse a command and execute it. pub fn try_execute_command(&mut self, mut buf: BytesMut) -> Option<(Command, String)> { let code = buf.get_u8() as char; @@ -363,6 +367,8 @@ impl QueryRouter { #[cfg(test)] mod test { + use std::collections::HashMap; + use super::*; use crate::messages::simple_query; use bytes::BufMut; @@ -370,7 +376,7 @@ mod test { #[test] fn test_defaults() { QueryRouter::setup(); - let qr = QueryRouter::new(ConnectionPool::default()); + let qr = QueryRouter::new(); assert_eq!(qr.role(), None); } @@ -378,7 +384,7 @@ mod test { #[test] fn test_infer_role_replica() { QueryRouter::setup(); - let mut qr = QueryRouter::new(ConnectionPool::default()); + let mut qr = QueryRouter::new(); assert!(qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")) != None); assert_eq!(qr.query_parser_enabled(), true); @@ -402,7 +408,7 @@ mod test { #[test] fn test_infer_role_primary() { QueryRouter::setup(); - let mut qr = QueryRouter::new(ConnectionPool::default()); + let mut qr = QueryRouter::new(); let queries = vec![ simple_query("UPDATE items SET name = 'pumpkin' WHERE id = 5"), @@ -421,7 +427,7 @@ mod test { #[test] fn test_infer_role_primary_reads_enabled() { QueryRouter::setup(); - let mut qr = QueryRouter::new(ConnectionPool::default()); + let mut qr = QueryRouter::new(); let query = simple_query("SELECT * FROM items WHERE id = 5"); assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO on")) != None); @@ -432,7 +438,7 @@ mod test { #[test] fn test_infer_role_parse_prepared() { QueryRouter::setup(); - let mut qr = QueryRouter::new(ConnectionPool::default()); + let mut qr = QueryRouter::new(); qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")); assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); @@ -523,7 +529,7 @@ mod test { #[test] fn test_try_execute_command() { QueryRouter::setup(); - let mut qr = QueryRouter::new(ConnectionPool::default()); + let mut qr = QueryRouter::new(); // SetShardingKey let query = simple_query("SET SHARDING KEY TO 13"); @@ -600,7 +606,7 @@ mod test { #[test] fn test_enable_query_parser() { QueryRouter::setup(); - let mut qr = QueryRouter::new(ConnectionPool::default()); + let mut qr = QueryRouter::new(); let query = simple_query("SET SERVER ROLE TO 'auto'"); assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); @@ -621,4 +627,43 @@ mod test { assert!(qr.try_execute_command(query) != None); assert!(qr.query_parser_enabled()); } + + #[test] + fn test_update_from_pool_settings() { + QueryRouter::setup(); + + let pool_settings = PoolSettings { + pool_mode: "transaction".to_string(), + shards: HashMap::default(), + user: crate::config::User::default(), + default_role: Role::Replica.to_string(), + query_parser_enabled: true, + primary_reads_enabled: false, + sharding_function: "pg_bigint_hash".to_string(), + }; + let mut qr = QueryRouter::new(); + assert_eq!(qr.active_role, None); + assert_eq!(qr.active_shard, None); + assert_eq!(qr.query_parser_enabled, false); + assert_eq!(qr.primary_reads_enabled, false); + + // Internal state must not be changed due to this, only defaults + qr.update_pool_settings(pool_settings.clone()); + + assert_eq!(qr.active_role, None); + assert_eq!(qr.active_shard, None); + assert_eq!(qr.query_parser_enabled, false); + assert_eq!(qr.primary_reads_enabled, false); + + let q1 = simple_query("SET SERVER ROLE TO 'primary'"); + assert!(qr.try_execute_command(q1) != None); + assert_eq!(qr.active_role.unwrap(), Role::Primary); + + let q2 = simple_query("SET SERVER ROLE TO 'default'"); + assert!(qr.try_execute_command(q2) != None); + assert_eq!( + qr.active_role.unwrap().to_string(), + pool_settings.clone().default_role + ); + } } diff --git a/tests/ruby/.ruby-version b/tests/ruby/.ruby-version index 860487ca1..cf232b52e 100644 --- a/tests/ruby/.ruby-version +++ b/tests/ruby/.ruby-version @@ -1 +1,2 @@ -2.7.1 +3.0.0 + diff --git a/tests/ruby/Gemfile b/tests/ruby/Gemfile index 05684c985..7b019183e 100644 --- a/tests/ruby/Gemfile +++ b/tests/ruby/Gemfile @@ -3,3 +3,4 @@ source "https://rubygems.org" gem "pg" gem "activerecord" gem "rubocop" +gem "toml", "~> 0.3.0" diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock index 607df18c2..3fd03471f 100644 --- a/tests/ruby/Gemfile.lock +++ b/tests/ruby/Gemfile.lock @@ -19,6 +19,7 @@ GEM parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) + parslet (2.0.0) pg (1.3.2) rainbow (3.1.1) regexp_parser (2.3.1) @@ -35,17 +36,21 @@ GEM rubocop-ast (1.17.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) + toml (0.3.0) + parslet (>= 1.8.0, < 3.0.0) tzinfo (2.0.4) concurrent-ruby (~> 1.0) unicode-display_width (2.1.0) PLATFORMS + arm64-darwin-21 x86_64-linux DEPENDENCIES activerecord pg rubocop + toml (~> 0.3.0) BUNDLED WITH 2.3.7 diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index c5a55a7e3..ba9476f43 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -2,6 +2,7 @@ require 'active_record' require 'pg' +require 'toml' $stdout.sync = true @@ -141,3 +142,62 @@ def test_server_parameters puts 'Server parameters ok' end + + +class ConfigEditor + def initialize + @original_config_text = File.read('../../.circleci/pgcat.toml') + text_to_load = @original_config_text.gsub("5432", "\"5432\"") + + @original_configs = TOML.load(text_to_load) + end + + def original_configs + TOML.load(TOML::Generator.new(@original_configs).body) + end + + def with_modified_configs(new_configs) + text_to_write = TOML::Generator.new(new_configs).body + text_to_write = text_to_write.gsub("\"5432\"", "5432") + File.write('../../.circleci/pgcat.toml', text_to_write) + yield + ensure + File.write('../../.circleci/pgcat.toml', @original_config_text) + end + +end + + +def test_reload_pool_recycling + admin_conn = PG::connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat") + server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") + + server_conn.async_exec("BEGIN") + conf_editor = ConfigEditor.new + new_configs = conf_editor.original_configs + + # swap shards + new_configs["pools"]["sharded_db"]["shards"]["0"]["database"] = "shard1" + new_configs["pools"]["sharded_db"]["shards"]["1"]["database"] = "shard0" + + raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard0' + conf_editor.with_modified_configs(new_configs) { admin_conn.async_exec("RELOAD") } + raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard0' + server_conn.async_exec("COMMIT;") + + # Transaction finished, client should get new configs + raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard1' + server_conn.close() + + # New connection should get new configs + server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") + raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard1' + +ensure + admin_conn.async_exec("RELOAD") # Go back to old state + admin_conn.close + server_conn.close + puts "Pool Recycling okay!" +end + +test_reload_pool_recycling From 5f1abbf379ba1db009871793a453fa9e70e9de2d Mon Sep 17 00:00:00 2001 From: Nicholas Dujay <3258756+dat2@users.noreply.github.com> Date: Tue, 9 Aug 2022 15:19:11 -0400 Subject: [PATCH 172/551] create a prometheus exporter on a standard http port (#107) * create a hyper server and add option to enable it in config * move prometheus stuff to its own file; update format * create metric type and help lookup table * finish the metric help type map * switch to a boolean and a standard port * dont emit unimplemented metrics * fail if curl returns a non 200 * resolve conflicts * move log out of config.show and into main * terminating new line * upgrade curl * include unimplemented stats --- .circleci/config.yml | 2 +- .circleci/pgcat.toml | 3 + .circleci/run_tests.sh | 3 + Cargo.lock | 247 +++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 + docker-compose.yml | 1 + examples/docker/pgcat.toml | 3 + pgcat.toml | 5 +- src/config.rs | 2 + src/main.rs | 27 +++- src/prometheus.rs | 212 +++++++++++++++++++++++++++++++ 11 files changed, 500 insertions(+), 7 deletions(-) create mode 100644 src/prometheus.rs diff --git a/.circleci/config.yml b/.circleci/config.yml index aed5c182d..337dcdc41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: command: "cargo fmt --check" - run: name: "Install dependencies" - command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip lcov llvm-11" + command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip lcov llvm-11 && sudo apt-get upgrade curl" - run: name: "Install rust tools" command: "cargo install cargo-binutils rustfilt && rustup component add llvm-tools-preview" diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 502215b06..bc37a2919 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -11,6 +11,9 @@ host = "0.0.0.0" # Port to run on, same as PgBouncer used in this example. port = 6432 +# enable prometheus exporter on port 9930 +enable_prometheus_exporter = true + # How long to wait before aborting a server connection (ms). connect_timeout = 100 diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 431f2d604..12e203648 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -31,6 +31,9 @@ toxiproxy-cli create -l 127.0.0.1:5433 -u 127.0.0.1:5432 postgres_replica start_pgcat "info" +# Check that prometheus is running +curl --fail localhost:9930/metrics + export PGPASSWORD=sharding_user export PGDATABASE=sharded_db diff --git a/Cargo.lock b/Cargo.lock index ddab730d8..2e20f7095 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,6 +159,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "futures-channel" version = "0.3.19" @@ -174,6 +180,12 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7" +[[package]] +name = "futures-sink" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" + [[package]] name = "futures-task" version = "0.3.19" @@ -215,6 +227,31 @@ dependencies = [ "wasi", ] +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -233,12 +270,80 @@ dependencies = [ "digest", ] +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "indexmap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "instant" version = "0.1.12" @@ -248,6 +353,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "itoa" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" + [[package]] name = "js-sys" version = "0.3.58" @@ -405,11 +516,13 @@ dependencies = [ "chrono", "env_logger", "hmac", + "hyper", "log", "md-5", "num_cpus", "once_cell", "parking_lot", + "phf", "rand", "regex", "rustls-pemfile", @@ -424,6 +537,50 @@ dependencies = [ "toml", ] +[[package]] +name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_macros", + "phf_shared", + "proc-macro-hack", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.8" @@ -442,6 +599,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.36" @@ -626,6 +789,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "slab" version = "0.4.5" @@ -638,6 +807,16 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "spin" version = "0.5.2" @@ -756,6 +935,20 @@ dependencies = [ "webpki", ] +[[package]] +name = "tokio-util" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.5.8" @@ -765,6 +958,50 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -804,6 +1041,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 8bdeab676..373702440 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,3 +31,5 @@ base64 = "0.13" stringprep = "0.1" tokio-rustls = "0.23" rustls-pemfile = "1" +hyper = { version = "0.14", features = ["full"] } +phf = { version = "0.10", features = ["macros"] } diff --git a/docker-compose.yml b/docker-compose.yml index 510546e61..89cb67ee3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,3 +14,4 @@ services: - "${PWD}/examples/docker/pgcat.toml:/etc/pgcat/pgcat.toml" ports: - "6432:6432" + - "9090:9090" diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index 40a54928c..cbfb1d9bc 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -11,6 +11,9 @@ host = "0.0.0.0" # Port to run on, same as PgBouncer used in this example. port = 6432 +# enable prometheus exporter on port 9930 +enable_prometheus_exporter = true + # How long to wait before aborting a server connection (ms). connect_timeout = 5000 diff --git a/pgcat.toml b/pgcat.toml index 50d797f81..d826994a0 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -11,6 +11,9 @@ host = "0.0.0.0" # Port to run on, same as PgBouncer used in this example. port = 6432 +# enable prometheus exporter on port 9930 +enable_prometheus_exporter = true + # How long to wait before aborting a server connection (ms). connect_timeout = 5000 @@ -61,7 +64,7 @@ query_parser_enabled = true # If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for # load balancing of read queries. Otherwise, the primary will only be used for write -# queries. The primary can always be explicitely selected with our custom protocol. +# queries. The primary can always be explicitly selected with our custom protocol. primary_reads_enabled = true # So what if you wanted to implement a different hashing function, diff --git a/src/config.rs b/src/config.rs index 05a1b3672..17aad85c2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -117,6 +117,7 @@ impl Default for User { pub struct General { pub host: String, pub port: i16, + pub enable_prometheus_exporter: Option, pub connect_timeout: u64, pub healthcheck_timeout: u64, pub shutdown_timeout: u64, @@ -133,6 +134,7 @@ impl Default for General { General { host: String::from("localhost"), port: 5432, + enable_prometheus_exporter: Some(false), connect_timeout: 5000, healthcheck_timeout: 1000, shutdown_timeout: 60000, diff --git a/src/main.rs b/src/main.rs index 5e5c9248e..7ae71feef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,6 +45,8 @@ use tokio::{ }; use std::collections::HashMap; +use std::net::SocketAddr; +use std::str::FromStr; use std::sync::Arc; use tokio::sync::broadcast; @@ -55,6 +57,7 @@ mod constants; mod errors; mod messages; mod pool; +mod prometheus; mod query_router; mod scram; mod server; @@ -62,11 +65,10 @@ mod sharding; mod stats; mod tls; -use config::{get_config, reload_config}; -use pool::{ClientServerMap, ConnectionPool}; -use stats::{Collector, Reporter, REPORTER}; - -use crate::config::VERSION; +use crate::config::{get_config, reload_config, VERSION}; +use crate::pool::{ClientServerMap, ConnectionPool}; +use crate::prometheus::start_metric_server; +use crate::stats::{Collector, Reporter, REPORTER}; #[tokio::main(worker_threads = 4)] async fn main() { @@ -95,6 +97,21 @@ async fn main() { }; let config = get_config(); + + if let Some(true) = config.general.enable_prometheus_exporter { + let http_addr_str = format!("{}:{}", config.general.host, crate::prometheus::HTTP_PORT); + let http_addr = match SocketAddr::from_str(&http_addr_str) { + Ok(addr) => addr, + Err(err) => { + error!("Invalid http address: {}", err); + return; + } + }; + tokio::task::spawn(async move { + start_metric_server(http_addr).await; + }); + } + let addr = format!("{}:{}", config.general.host, config.general.port); let listener = match TcpListener::bind(&addr).await { diff --git a/src/prometheus.rs b/src/prometheus.rs new file mode 100644 index 000000000..eeaa9b15f --- /dev/null +++ b/src/prometheus.rs @@ -0,0 +1,212 @@ +use hyper::service::{make_service_fn, service_fn}; +use hyper::{Body, Method, Request, Response, Server, StatusCode}; +use log::{error, info, warn}; +use phf::phf_map; +use std::collections::HashMap; +use std::fmt; +use std::net::SocketAddr; + +use crate::config::Address; +use crate::pool::get_all_pools; +use crate::stats::get_stats; + +pub const HTTP_PORT: usize = 9930; + +struct MetricHelpType { + help: &'static str, + ty: &'static str, +} + +// reference for metric types: https://prometheus.io/docs/concepts/metric_types/ +// counters only increase +// gauges can arbitrarily increase or decrease +static METRIC_HELP_AND_TYPES_LOOKUP: phf::Map<&'static str, MetricHelpType> = phf_map! { + "total_query_count" => MetricHelpType { + help: "Number of queries sent by all clients", + ty: "counter", + }, + "total_query_time" => MetricHelpType { + help: "Total amount of time for queries to execute", + ty: "counter", + }, + "total_received" => MetricHelpType { + help: "Number of bytes received from the server", + ty: "counter", + }, + "total_sent" => MetricHelpType { + help: "Number of bytes sent to the server", + ty: "counter", + }, + "total_xact_count" => MetricHelpType { + help: "Total number of transactions started by the client", + ty: "counter", + }, + "total_xact_time" => MetricHelpType { + help: "Total amount of time for all transactions to execute", + ty: "counter", + }, + "total_wait_time" => MetricHelpType { + help: "Total time client waited for a server connection", + ty: "counter", + }, + "avg_query_count" => MetricHelpType { + help: "Average of total_query_count every 15 seconds", + ty: "gauge", + }, + "avg_query_time" => MetricHelpType { + help: "Average time taken for queries to execute every 15 seconds", + ty: "gauge", + }, + "avg_recv" => MetricHelpType { + help: "Average of total_received bytes every 15 seconds", + ty: "gauge", + }, + "avg_sent" => MetricHelpType { + help: "Average of total_sent bytes every 15 seconds", + ty: "gauge", + }, + "avg_xact_count" => MetricHelpType { + help: "Average of total_xact_count every 15 seconds", + ty: "gauge", + }, + "avg_xact_time" => MetricHelpType { + help: "Average of total_xact_time every 15 seconds", + ty: "gauge", + }, + "avg_wait_time" => MetricHelpType { + help: "Average of total_wait_time every 15 seconds", + ty: "gauge", + }, + "maxwait_us" => MetricHelpType { + help: "The time a client waited for a server connection in microseconds", + ty: "gauge", + }, + "maxwait" => MetricHelpType { + help: "The time a client waited for a server connection in seconds", + ty: "gauge", + }, + "cl_waiting" => MetricHelpType { + help: "How many clients are waiting for a connection from the pool", + ty: "gauge", + }, + "cl_active" => MetricHelpType { + help: "How many clients are actively communicating with a server", + ty: "gauge", + }, + "cl_idle" => MetricHelpType { + help: "How many clients are idle", + ty: "gauge", + }, + "sv_idle" => MetricHelpType { + help: "How many server connections are idle", + ty: "gauge", + }, + "sv_active" => MetricHelpType { + help: "How many server connections are actively communicating with a client", + ty: "gauge", + }, + "sv_login" => MetricHelpType { + help: "How many server connections are currently being created", + ty: "gauge", + }, + "sv_tested" => MetricHelpType { + help: "How many server connections are currently waiting on a health check to succeed", + ty: "gauge", + }, +}; + +struct PrometheusMetric { + name: String, + help: String, + ty: String, + labels: HashMap<&'static str, String>, + value: i64, +} + +impl fmt::Display for PrometheusMetric { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let formatted_labels = self + .labels + .iter() + .map(|(key, value)| format!("{}=\"{}\"", key, value)) + .collect::>() + .join(","); + write!( + f, + "# HELP {name} {help}\n# TYPE {name} {ty}\n{name}{{{formatted_labels}}} {value}\n", + name = format_args!("pgcat_{}", self.name), + help = self.help, + ty = self.ty, + formatted_labels = formatted_labels, + value = self.value + ) + } +} + +impl PrometheusMetric { + fn new(address: &Address, name: &str, value: i64) -> Option { + let mut labels = HashMap::new(); + labels.insert("host", address.host.clone()); + labels.insert("shard", address.shard.to_string()); + labels.insert("role", address.role.to_string()); + labels.insert("database", address.database.to_string()); + + METRIC_HELP_AND_TYPES_LOOKUP + .get(name) + .map(|metric| PrometheusMetric { + name: name.to_owned(), + help: metric.help.to_owned(), + ty: metric.ty.to_owned(), + labels, + value, + }) + } +} + +async fn prometheus_stats(request: Request) -> Result, hyper::http::Error> { + match (request.method(), request.uri().path()) { + (&Method::GET, "/metrics") => { + let stats = get_stats(); + + let mut lines = Vec::new(); + for (_, pool) in get_all_pools() { + for shard in 0..pool.shards() { + for server in 0..pool.servers(shard) { + let address = pool.address(shard, server); + if let Some(address_stats) = stats.get(&address.id) { + for (key, value) in address_stats.iter() { + if let Some(prometheus_metric) = + PrometheusMetric::new(address, key, *value) + { + lines.push(prometheus_metric.to_string()); + } else { + warn!("Metric {} not implemented for {}", key, address.name()); + } + } + } + } + } + } + + Response::builder() + .header("content-type", "text/plain; version=0.0.4") + .body(lines.join("\n").into()) + } + _ => Response::builder() + .status(StatusCode::NOT_FOUND) + .body("".into()), + } +} + +pub async fn start_metric_server(http_addr: SocketAddr) { + let http_service_factory = + make_service_fn(|_conn| async { Ok::<_, hyper::Error>(service_fn(prometheus_stats)) }); + let server = Server::bind(&http_addr.into()).serve(http_service_factory); + info!( + "Exposing prometheus metrics on http://{}/metrics.", + http_addr + ); + if let Err(e) = server.await { + error!("Failed to run HTTP server: {}.", e); + } +} From 15fb023568458b2c7883f78fe8ecae7385752c6e Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Tue, 9 Aug 2022 17:09:53 -0400 Subject: [PATCH 173/551] Validates pgcat is closed after shutdown python tests (#116) * Validates pgcat is closed after shutdown python tests * Fix pgrep logic * Moves sigterm step to after cleanup to decouple * Replace subprocess with os.system for running pgcat --- tests/python/tests.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/python/tests.py b/tests/python/tests.py index 3ff99a091..a674cee67 100644 --- a/tests/python/tests.py +++ b/tests/python/tests.py @@ -3,8 +3,6 @@ import psutil import os import signal -import subprocess -from threading import Thread import time SHUTDOWN_TIMEOUT = 5 @@ -15,14 +13,18 @@ def pgcat_start(): pg_cat_send_signal(signal.SIGTERM) - pgcat_start_command = "./target/debug/pgcat .circleci/pgcat.toml" - subprocess.Popen(pgcat_start_command.split()) + os.system("./target/debug/pgcat .circleci/pgcat.toml &") def pg_cat_send_signal(signal: signal.Signals): for proc in psutil.process_iter(["pid", "name"]): if "pgcat" == proc.name(): os.kill(proc.pid, signal) + if signal == signal.SIGTERM: + # Returns 0 if pgcat process exists + time.sleep(2) + if not os.system('pgrep pgcat'): + raise Exception("pgcat not closed after SIGTERM") def connect_normal_db( @@ -67,8 +69,7 @@ def test_shutdown_logic(): ##### NO ACTIVE QUERIES SIGINT HANDLING ##### # Start pgcat - server = Thread(target=pgcat_start) - server.start() + pgcat_start() # Wait for server to fully start up time.sleep(2) @@ -92,12 +93,15 @@ def test_shutdown_logic(): else: # Fail if query execution succeeded raise Exception("Server not closed after sigint") + cleanup_conn(conn, cur) + pg_cat_send_signal(signal.SIGTERM) + + ##### END ##### ##### HANDLE TRANSACTION WITH SIGINT ##### # Start pgcat - server = Thread(target=pgcat_start) - server.start() + pgcat_start() # Wait for server to fully start up time.sleep(2) @@ -120,11 +124,13 @@ def test_shutdown_logic(): raise Exception("Server closed while in transaction", e.pgerror) cleanup_conn(conn, cur) + pg_cat_send_signal(signal.SIGTERM) + + ##### END ##### ##### HANDLE SHUTDOWN TIMEOUT WITH SIGINT ##### # Start pgcat - server = Thread(target=pgcat_start) - server.start() + pgcat_start() # Wait for server to fully start up time.sleep(3) @@ -151,6 +157,9 @@ def test_shutdown_logic(): raise Exception("Server not closed after sigint and expected timeout") cleanup_conn(conn, cur) + pg_cat_send_signal(signal.SIGTERM) + + ##### END ##### test_normal_db_access() From 237a876f7dc02b2a99c7c4d394df1ec7744ced74 Mon Sep 17 00:00:00 2001 From: Nicholas Dujay <3258756+dat2@users.noreply.github.com> Date: Tue, 9 Aug 2022 17:15:34 -0400 Subject: [PATCH 174/551] fix docker compose port allocation for local dev (#117) change docker compose port to right prometheus port --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 89cb67ee3..e93d8eb95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,4 +14,4 @@ services: - "${PWD}/examples/docker/pgcat.toml:/etc/pgcat/pgcat.toml" ports: - "6432:6432" - - "9090:9090" + - "9930:9930" From d841a43f5e38a71ef1506ca05bcd37659cfa756a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 10 Aug 2022 09:51:56 -0700 Subject: [PATCH 175/551] Update CONTRIBUTING.md --- CONTRIBUTING.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b1d8b379b..10d4924a7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,10 +10,4 @@ Happy hacking! ## TODOs -A non-exhaustive list of things that would be useful to implement: - -#### Client authentication -MD5 is probably sufficient, but maybe others too. - -#### Admin -Admin database for stats collection and pooler administration. PgBouncer gives us a nice example on how to do that, specifically how to implement `RowDescription` and `DataRow` messages, [example here](https://github.com/pgbouncer/pgbouncer/blob/4f9ced8e63d317a6ff45c8b0efa876b32161f6db/src/admin.c#L813). +See [Issues]([url](https://github.com/levkk/pgcat/issues)). From 35eff736f434c42739815e05b7bf5ed44abd2c89 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 11 Aug 2022 17:42:40 -0400 Subject: [PATCH 176/551] Health check delay (#118) * initial commit of server check delay implementation * fmt * spelling * Update name to last_healthcheck and some comments * Moved server tested stat to after require_healthcheck check * Make health check delay configurable * Rename to last_activity * Fix typo * Add debug log for healthcheck * Add address to debug log --- .circleci/pgcat.toml | 3 + README.md | 2 + examples/docker/pgcat.toml | 3 + pgcat.toml | 3 + src/client.rs | 131 +++++++++++++++++++++++++++++-------- src/config.rs | 7 ++ src/pool.rs | 36 ++++++---- src/server.rs | 21 +++++- 8 files changed, 165 insertions(+), 41 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index bc37a2919..24dfe97da 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -20,6 +20,9 @@ connect_timeout = 100 # How much time to give the health check query to return with a result (ms). healthcheck_timeout = 100 +# How long to keep connection available for immediate re-use, without running a healthcheck query on it +healthcheck_delay = 30000 + # How much time to give clients during shutdown before forcibly killing client connections (ms). shutdown_timeout = 5000 diff --git a/README.md b/README.md index c03982d2b..1fac2b601 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ psql -h 127.0.0.1 -p 6432 -c 'SELECT 1' | `connect_timeout` | Maximum time to establish a connection to a server (milliseconds). If reached, the server is banned and the next target is attempted. | `5000` | | `healthcheck_timeout` | Maximum time to pass a health check (`SELECT 1`, milliseconds). If reached, the server is banned and the next target is attempted. | `1000` | | `shutdown_timeout` | Maximum time to give clients during shutdown before forcibly killing client connections (ms). | `60000` | +| `healthcheck_delay` | How long to keep connection available for immediate re-use, without running a healthcheck query on it | `30000` | | `ban_time` | Ban time for a server (seconds). It won't be allowed to serve transactions until the ban expires; failover targets will be used instead. | `60` | | | | | | **`user`** | | | @@ -252,6 +253,7 @@ The config can be reloaded by sending a `kill -s SIGHUP` to the process or by qu | `connect_timeout` | yes | | `healthcheck_timeout` | no | | `shutdown_timeout` | no | +| `healthcheck_delay` | no | | `ban_time` | no | | `user` | yes | | `shards` | yes | diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index cbfb1d9bc..b2ce0b64e 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -20,6 +20,9 @@ connect_timeout = 5000 # How much time to give `SELECT 1` health check query to return with a result (ms). healthcheck_timeout = 1000 +# How long to keep connection available for immediate re-use, without running a healthcheck query on it +healthcheck_delay = 30000 + # How much time to give clients during shutdown before forcibly killing client connections (ms). shutdown_timeout = 60000 diff --git a/pgcat.toml b/pgcat.toml index d826994a0..5b0467854 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -20,6 +20,9 @@ connect_timeout = 5000 # How much time to give the health check query to return with a result (ms). healthcheck_timeout = 1000 +# How long to keep connection available for immediate re-use, without running a healthcheck query on it +healthcheck_delay = 30000 + # How much time to give clients during shutdown before forcibly killing client connections (ms). shutdown_timeout = 60000 diff --git a/src/client.rs b/src/client.rs index 9d4f40389..1dd1bccbc 100644 --- a/src/client.rs +++ b/src/client.rs @@ -7,11 +7,11 @@ use tokio::net::TcpStream; use tokio::sync::broadcast::Receiver; use crate::admin::{generate_server_info_for_admin, handle_admin}; -use crate::config::get_config; +use crate::config::{get_config, Address}; use crate::constants::*; use crate::errors::Error; use crate::messages::*; -use crate::pool::{get_pool, ClientServerMap}; +use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; use crate::stats::{get_reporter, Reporter}; @@ -246,7 +246,7 @@ where } } -/// Handle TLS connection negotation. +/// Handle TLS connection negotiation. pub async fn startup_tls( stream: TcpStream, client_server_map: ClientServerMap, @@ -259,14 +259,14 @@ pub async fn startup_tls( let mut stream = match tls.acceptor.accept(stream).await { Ok(stream) => stream, - // TLS negotitation failed. + // TLS negotiation failed. Err(err) => { error!("TLS negotiation failed: {:?}", err); return Err(Error::TlsError); } }; - // TLS negotitation successful. + // TLS negotiation successful. // Continue with regular startup using encrypted connection. match get_startup::>(&mut stream).await { // Got good startup message, proceeding like normal except we @@ -540,21 +540,21 @@ where // Get a pool instance referenced by the most up-to-date // pointer. This ensures we always read the latest config // when starting a query. - let mut pool = - match get_pool(self.target_pool_name.clone(), self.target_user_name.clone()) { - Some(pool) => pool, - None => { - error_response( - &mut self.write, - &format!( - "No pool configured for database: {:?}, user: {:?}", - self.target_pool_name, self.target_user_name - ), - ) - .await?; - return Err(Error::ClientError); - } - }; + let pool = match get_pool(self.target_pool_name.clone(), self.target_user_name.clone()) + { + Some(pool) => pool, + None => { + error_response( + &mut self.write, + &format!( + "No pool configured for database: {:?}, user: {:?}", + self.target_pool_name, self.target_user_name + ), + ) + .await?; + return Err(Error::ClientError); + } + }; query_router.update_pool_settings(pool.settings.clone()); let current_shard = query_router.shard(); @@ -731,12 +731,26 @@ where 'Q' => { debug!("Sending query to server"); - server.send(original).await?; + self.send_server_message( + server, + original, + &address, + query_router.shard(), + &pool, + ) + .await?; // Read all data the server has to offer, which can be multiple messages // buffered in 8196 bytes chunks. loop { - let response = server.recv().await?; + let response = self + .receive_server_message( + server, + &address, + query_router.shard(), + &pool, + ) + .await?; // Send server reply to the client. match write_all_half(&mut self.write, response).await { @@ -816,14 +830,28 @@ where self.buffer.put(&original[..]); - server.send(self.buffer.clone()).await?; + self.send_server_message( + server, + self.buffer.clone(), + &address, + query_router.shard(), + &pool, + ) + .await?; self.buffer.clear(); // Read all data the server has to offer, which can be multiple messages // buffered in 8196 bytes chunks. loop { - let response = server.recv().await?; + let response = self + .receive_server_message( + server, + &address, + query_router.shard(), + &pool, + ) + .await?; match write_all_half(&mut self.write, response).await { Ok(_) => (), @@ -857,15 +885,31 @@ where 'd' => { // Forward the data to the server, // don't buffer it since it can be rather large. - server.send(original).await?; + self.send_server_message( + server, + original, + &address, + query_router.shard(), + &pool, + ) + .await?; } // CopyDone or CopyFail // Copy is done, successfully or not. 'c' | 'f' => { - server.send(original).await?; + self.send_server_message( + server, + original, + &address, + query_router.shard(), + &pool, + ) + .await?; - let response = server.recv().await?; + let response = self + .receive_server_message(server, &address, query_router.shard(), &pool) + .await?; match write_all_half(&mut self.write, response).await { Ok(_) => (), @@ -907,6 +951,39 @@ where let mut guard = self.client_server_map.lock(); guard.remove(&(self.process_id, self.secret_key)); } + + async fn send_server_message( + &self, + server: &mut Server, + message: BytesMut, + address: &Address, + shard: usize, + pool: &ConnectionPool, + ) -> Result<(), Error> { + match server.send(message).await { + Ok(_) => Ok(()), + Err(err) => { + pool.ban(address, shard, self.process_id); + Err(err) + } + } + } + + async fn receive_server_message( + &self, + server: &mut Server, + address: &Address, + shard: usize, + pool: &ConnectionPool, + ) -> Result { + match server.recv().await { + Ok(message) => Ok(message), + Err(err) => { + pool.ban(address, shard, self.process_id); + Err(err) + } + } + } } impl Drop for Client { diff --git a/src/config.rs b/src/config.rs index 17aad85c2..435cd9e3b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -121,6 +121,7 @@ pub struct General { pub connect_timeout: u64, pub healthcheck_timeout: u64, pub shutdown_timeout: u64, + pub healthcheck_delay: u64, pub ban_time: i64, pub autoreload: bool, pub tls_certificate: Option, @@ -138,6 +139,7 @@ impl Default for General { connect_timeout: 5000, healthcheck_timeout: 1000, shutdown_timeout: 60000, + healthcheck_delay: 30000, ban_time: 60, autoreload: false, tls_certificate: None, @@ -281,6 +283,10 @@ impl From<&Config> for std::collections::HashMap { "shutdown_timeout".to_string(), config.general.shutdown_timeout.to_string(), ), + ( + "healthcheck_delay".to_string(), + config.general.healthcheck_delay.to_string(), + ), ("ban_time".to_string(), config.general.ban_time.to_string()), ]; @@ -299,6 +305,7 @@ impl Config { ); info!("Connection timeout: {}ms", self.general.connect_timeout); info!("Shutdown timeout: {}ms", self.general.shutdown_timeout); + info!("Healthcheck delay: {}ms", self.general.healthcheck_delay); match self.general.tls_certificate.clone() { Some(tls_certificate) => { info!("TLS certificate: {}", tls_certificate); diff --git a/src/pool.rs b/src/pool.rs index 775c8d42b..fcb6f855d 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -251,7 +251,7 @@ impl ConnectionPool { /// Get a connection from the pool. pub async fn get( - &mut self, + &self, shard: usize, // shard number role: Option, // primary or replica process_id: i32, // client id @@ -283,6 +283,9 @@ impl ConnectionPool { return Err(Error::BadConfig); } + let healthcheck_timeout = get_config().general.healthcheck_timeout; + let healthcheck_delay = get_config().general.healthcheck_delay as u128; + while allowed_attempts > 0 { // Round-robin replicas. round_robin += 1; @@ -312,7 +315,7 @@ impl ConnectionPool { Ok(conn) => conn, Err(err) => { error!("Banning replica {}, error: {:?}", index, err); - self.ban(address, shard); + self.ban(address, shard, process_id); self.stats.client_disconnecting(process_id, address.id); self.stats .checkout_time(now.elapsed().as_micros(), process_id, address.id); @@ -322,8 +325,19 @@ impl ConnectionPool { // // Check if this server is alive with a health check. let server = &mut *conn; - let healthcheck_timeout = get_config().general.healthcheck_timeout; + // Will return error if timestamp is greater than current system time, which it should never be set to + let require_healthcheck = + server.last_activity().elapsed().unwrap().as_millis() > healthcheck_delay; + + if !require_healthcheck { + self.stats + .checkout_time(now.elapsed().as_micros(), process_id, address.id); + self.stats.server_idle(conn.process_id(), address.id); + return Ok((conn, address.clone())); + } + + debug!("Running health check for replica {}, {:?}", index, address); self.stats.server_tested(server.process_id(), address.id); match tokio::time::timeout( @@ -348,10 +362,7 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(address, shard); - self.stats.client_disconnecting(process_id, address.id); - self.stats - .checkout_time(now.elapsed().as_micros(), process_id, address.id); + self.ban(address, shard, process_id); continue; } }, @@ -362,10 +373,7 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(address, shard); - self.stats.client_disconnecting(process_id, address.id); - self.stats - .checkout_time(now.elapsed().as_micros(), process_id, address.id); + self.ban(address, shard, process_id); continue; } } @@ -377,7 +385,11 @@ impl ConnectionPool { /// Ban an address (i.e. replica). It no longer will serve /// traffic for any new transactions. Existing transactions on that replica /// will finish successfully or error out to the clients. - pub fn ban(&self, address: &Address, shard: usize) { + pub fn ban(&self, address: &Address, shard: usize, process_id: i32) { + self.stats.client_disconnecting(process_id, address.id); + self.stats + .checkout_time(Instant::now().elapsed().as_micros(), process_id, address.id); + error!("Banning {:?}", address); let now = chrono::offset::Utc::now().naive_utc(); let mut guard = self.banlist.write(); diff --git a/src/server.rs b/src/server.rs index d21696f0b..ddf95ce62 100644 --- a/src/server.rs +++ b/src/server.rs @@ -2,6 +2,7 @@ /// Here we are pretending to the a Postgres client. use bytes::{Buf, BufMut, BytesMut}; use log::{debug, error, info, trace}; +use std::time::SystemTime; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ tcp::{OwnedReadHalf, OwnedWriteHalf}, @@ -58,6 +59,9 @@ pub struct Server { /// Application name using the server at the moment. application_name: String, + + // Last time that a successful server send or response happened + last_activity: SystemTime, } impl Server { @@ -316,6 +320,7 @@ impl Server { connected_at: chrono::offset::Utc::now().naive_utc(), stats: stats, application_name: String::new(), + last_activity: SystemTime::now(), }; server.set_name("pgcat").await?; @@ -366,7 +371,11 @@ impl Server { .data_sent(messages.len(), self.process_id, self.address.id); match write_all_half(&mut self.write, messages).await { - Ok(_) => Ok(()), + Ok(_) => { + // Successfully sent to server + self.last_activity = SystemTime::now(); + Ok(()) + } Err(err) => { error!("Terminating server because of: {:?}", err); self.bad = true; @@ -413,7 +422,7 @@ impl Server { self.in_transaction = false; } - // Some error occured, the transaction was rolled back. + // Some error occurred, the transaction was rolled back. 'E' => { self.in_transaction = true; } @@ -474,6 +483,9 @@ impl Server { // Clear the buffer for next query. self.buffer.clear(); + // Successfully received data from server + self.last_activity = SystemTime::now(); + // Pass the data back to the client. Ok(bytes) } @@ -564,6 +576,11 @@ impl Server { pub fn process_id(&self) -> i32 { self.process_id } + + // Get server's latest response timestamp + pub fn last_activity(&self) -> SystemTime { + self.last_activity + } } impl Drop for Server { From 4e91331807f81aab090a69f0b6bdf1a011580352 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 11 Aug 2022 22:41:08 -0700 Subject: [PATCH 177/551] Speed up CI a bit (#119) * Sleep for 1s * use premade image * quicker * revert shutdown timeout --- .circleci/config.yml | 2 +- .circleci/run_tests.sh | 2 ++ Dockerfile.ci | 8 ++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.ci diff --git a/.circleci/config.yml b/.circleci/config.yml index 337dcdc41..1d3449a1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - - image: cimg/rust:1.62.0 + - image: levkk/pgcat-ci:latest environment: RUST_LOG: info RUSTFLAGS: "-C instrument-coverage" diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 12e203648..835bd10ea 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -122,6 +122,8 @@ sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' .circleci/pgcat.toml # Reload config test kill -SIGHUP $(pgrep pgcat) +sleep 1 + # Prepared statements that will only work in session mode pgbench -U sharding_user -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol prepared diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 000000000..8f8ab9459 --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,8 @@ +FROM cimg/rust:1.62.0 +RUN sudo apt-get update && \ + sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip lcov llvm-11 && \ + sudo apt-get upgrade curl +RUN cargo install cargo-binutils rustfilt && \ + rustup component add llvm-tools-preview +RUN pip3 install psycopg2 && \ + sudo gem install bundler From 04a1f37a10e30ded0d954f6aa7e4eb410ef079dd Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 11 Aug 2022 22:47:47 -0700 Subject: [PATCH 178/551] Fix debug log (#120) --- src/pool.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pool.rs b/src/pool.rs index fcb6f855d..76445c8fb 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -337,7 +337,8 @@ impl ConnectionPool { return Ok((conn, address.clone())); } - debug!("Running health check for replica {}, {:?}", index, address); + debug!("Running health check on server {:?}", address); + self.stats.server_tested(server.process_id(), address.id); match tokio::time::timeout( From a6f93df476ba541daa1c09dc592a6a1c9f590664 Mon Sep 17 00:00:00 2001 From: Pradeep Chhetri <30620077+chhetripradeep@users.noreply.github.com> Date: Sun, 14 Aug 2022 01:25:14 +0800 Subject: [PATCH 179/551] Make prometheus port configurable (#121) * Make prometheus port configurable * Update circleci config --- .circleci/pgcat.toml | 5 +++- README.md | 52 ++++++++++++++++++++------------------ docker-compose.yml | 2 +- examples/docker/pgcat.toml | 5 +++- pgcat.toml | 5 +++- src/config.rs | 6 +++++ src/main.rs | 5 +++- src/prometheus.rs | 2 -- 8 files changed, 51 insertions(+), 31 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 24dfe97da..9c3babf2c 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -11,9 +11,12 @@ host = "0.0.0.0" # Port to run on, same as PgBouncer used in this example. port = 6432 -# enable prometheus exporter on port 9930 +# Whether to enable prometheus exporter or not. enable_prometheus_exporter = true +# Port at which prometheus exporter listens on. +prometheus_exporter_port = 9930 + # How long to wait before aborting a server connection (ms). connect_timeout = 100 diff --git a/README.md b/README.md index 1fac2b601..90cbd823a 100644 --- a/README.md +++ b/README.md @@ -38,30 +38,34 @@ psql -h 127.0.0.1 -p 6432 -c 'SELECT 1' ### Config -| **Name** | **Description** | **Examples** | -|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------| -| **`general`** | | | -| `host` | The pooler will run on this host, 0.0.0.0 means accessible from everywhere. | `0.0.0.0` | -| `port` | The pooler will run on this port. | `6432` | -| `pool_size` | Maximum allowed server connections per pool. Pools are separated for each user/shard/server role. The connections are allocated as needed. | `15` | -| `pool_mode` | The pool mode to use, i.e. `session` or `transaction`. | `transaction` | -| `connect_timeout` | Maximum time to establish a connection to a server (milliseconds). If reached, the server is banned and the next target is attempted. | `5000` | -| `healthcheck_timeout` | Maximum time to pass a health check (`SELECT 1`, milliseconds). If reached, the server is banned and the next target is attempted. | `1000` | -| `shutdown_timeout` | Maximum time to give clients during shutdown before forcibly killing client connections (ms). | `60000` | -| `healthcheck_delay` | How long to keep connection available for immediate re-use, without running a healthcheck query on it | `30000` | -| `ban_time` | Ban time for a server (seconds). It won't be allowed to serve transactions until the ban expires; failover targets will be used instead. | `60` | -| | | | -| **`user`** | | | -| `name` | The user name. | `sharding_user` | -| `password` | The user password in plaintext. | `hunter2` | -| | | | -| **`shards`** | Shards are numerically numbered starting from 0; the order in the config is preserved by the pooler to route queries accordingly. | `[shards.0]` | -| `servers` | List of servers to connect to and their roles. A server is: `[host, port, role]`, where `role` is either `primary` or `replica`. | `["127.0.0.1", 5432, "primary"]` | -| `database` | The name of the database to connect to. This is the same on all servers that are part of one shard. | | -| **`query_router`** | | | -| `default_role` | Traffic is routed to this role by default (round-robin), unless the client specifies otherwise. Default is `any`, for any role available. | `any`, `primary`, `replica` | -| `query_parser_enabled` | Enable the query parser which will inspect incoming queries and route them to a primary or replicas. | `false` | -| `primary_reads_enabled` | Enable this to allow read queries on the primary; otherwise read queries are routed to the replicas. | `true` | +| **Name** | **Description** | **Examples** | +|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------| +| **`general`** | | | +| `host` | The pooler will run on this host, 0.0.0.0 means accessible from everywhere. | `0.0.0.0` | +| `port` | The pooler will run on this port. | `6432` | +| `enable_prometheus_exporter` | Enable prometheus exporter which will export metrics in prometheus exposition format. | `true` | +| `prometheus_exporter_port` | Port at which prometheus exporter listens on. | `9930` | +| `pool_size` | Maximum allowed server connections per pool. Pools are separated for each user/shard/server role. The connections are allocated as needed. | `15` | +| `pool_mode` | The pool mode to use, i.e. `session` or `transaction`. | `transaction` | +| `connect_timeout` | Maximum time to establish a connection to a server (milliseconds). If reached, the server is banned and the next target is attempted. | `5000` | +| `healthcheck_timeout` | Maximum time to pass a health check (`SELECT 1`, milliseconds). If reached, the server is banned and the next target is attempted. | `1000` | +| `shutdown_timeout` | Maximum time to give clients during shutdown before forcibly killing client connections (ms). | `60000` | +| `healthcheck_delay` | How long to keep connection available for immediate re-use, without running a healthcheck query on it | `30000` | +| `ban_time` | Ban time for a server (seconds). It won't be allowed to serve transactions until the ban expires; failover targets will be used instead. | `60` | +| `autoreload` | Enable auto-reload of config after fixed time-interval. | `false` | +| | | | +| **`user`** | | | +| `name` | The user name. | `sharding_user` | +| `password` | The user password in plaintext. | `hunter2` | +| | | | +| **`shards`** | Shards are numerically numbered starting from 0; the order in the config is preserved by the pooler to route queries accordingly. | `[shards.0]` | +| `servers` | List of servers to connect to and their roles. A server is: `[host, port, role]`, where `role` is either `primary` or `replica`. | `["127.0.0.1", 5432, "primary"]` | +| `database` | The name of the database to connect to. This is the same on all servers that are part of one shard. | | +| | | | +| **`query_router`** | | | +| `default_role` | Traffic is routed to this role by default (round-robin), unless the client specifies otherwise. Default is `any`, for any role available. | `any`, `primary`, `replica` | +| `query_parser_enabled` | Enable the query parser which will inspect incoming queries and route them to a primary or replicas. | `false` | +| `primary_reads_enabled` | Enable this to allow read queries on the primary; otherwise read queries are routed to the replicas. | `true` | ## Local development diff --git a/docker-compose.yml b/docker-compose.yml index e93d8eb95..96d1f3953 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: postgres: - image: postgres:13 + image: postgres:14 environment: POSTGRES_PASSWORD: postgres POSTGRES_HOST_AUTH_METHOD: md5 diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index b2ce0b64e..3c74df33c 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -11,9 +11,12 @@ host = "0.0.0.0" # Port to run on, same as PgBouncer used in this example. port = 6432 -# enable prometheus exporter on port 9930 +# Whether to enable prometheus exporter or not. enable_prometheus_exporter = true +# Port at which prometheus exporter listens on. +prometheus_exporter_port = 9930 + # How long to wait before aborting a server connection (ms). connect_timeout = 5000 diff --git a/pgcat.toml b/pgcat.toml index 5b0467854..bc246f493 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -11,9 +11,12 @@ host = "0.0.0.0" # Port to run on, same as PgBouncer used in this example. port = 6432 -# enable prometheus exporter on port 9930 +# Whether to enable prometheus exporter or not. enable_prometheus_exporter = true +# Port at which prometheus exporter listens on. +prometheus_exporter_port = 9930 + # How long to wait before aborting a server connection (ms). connect_timeout = 5000 diff --git a/src/config.rs b/src/config.rs index 435cd9e3b..57b52ae18 100644 --- a/src/config.rs +++ b/src/config.rs @@ -118,6 +118,7 @@ pub struct General { pub host: String, pub port: i16, pub enable_prometheus_exporter: Option, + pub prometheus_exporter_port: i16, pub connect_timeout: u64, pub healthcheck_timeout: u64, pub shutdown_timeout: u64, @@ -136,6 +137,7 @@ impl Default for General { host: String::from("localhost"), port: 5432, enable_prometheus_exporter: Some(false), + prometheus_exporter_port: 9930, connect_timeout: 5000, healthcheck_timeout: 1000, shutdown_timeout: 60000, @@ -271,6 +273,10 @@ impl From<&Config> for std::collections::HashMap { let mut static_settings = vec![ ("host".to_string(), config.general.host.to_string()), ("port".to_string(), config.general.port.to_string()), + ( + "prometheus_exporter_port".to_string(), + config.general.prometheus_exporter_port.to_string(), + ), ( "connect_timeout".to_string(), config.general.connect_timeout.to_string(), diff --git a/src/main.rs b/src/main.rs index 7ae71feef..db934747b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -99,7 +99,10 @@ async fn main() { let config = get_config(); if let Some(true) = config.general.enable_prometheus_exporter { - let http_addr_str = format!("{}:{}", config.general.host, crate::prometheus::HTTP_PORT); + let http_addr_str = format!( + "{}:{}", + config.general.host, config.general.prometheus_exporter_port + ); let http_addr = match SocketAddr::from_str(&http_addr_str) { Ok(addr) => addr, Err(err) => { diff --git a/src/prometheus.rs b/src/prometheus.rs index eeaa9b15f..4e3dc3aaa 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -10,8 +10,6 @@ use crate::config::Address; use crate::pool::get_all_pools; use crate::stats::get_stats; -pub const HTTP_PORT: usize = 9930; - struct MetricHelpType { help: &'static str, ty: &'static str, From 4e4857d6bdebe168cd17546d03b7ca8b50a7aaa7 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 13 Aug 2022 13:45:58 -0700 Subject: [PATCH 180/551] Statement timeout + replica imbalance fix (#122) * Statement timeout * send error message too * Correct error messages * Fix replica inbalance * disable stmt timeout by default * Redundant mark_bad * revert healthcheck delay * tests * set it to 0 * reload config again --- .circleci/pgcat.toml | 3 +++ .circleci/run_tests.sh | 12 ++++++++++ pgcat.toml | 5 +++++ src/client.rs | 51 ++++++++++++++++++++++++++++++++++++------ src/config.rs | 12 ++++++++++ src/errors.rs | 1 + 6 files changed, 77 insertions(+), 7 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 9c3babf2c..56aa1ddc2 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -91,11 +91,13 @@ password = "sharding_user" # The maximum number of connection from a single Pgcat process to any database in the cluster # is the sum of pool_size across all users. pool_size = 9 +statement_timeout = 0 [pools.sharded_db.users.1] username = "other_user" password = "other_user" pool_size = 21 +statement_timeout = 30000 # Shard 0 [pools.sharded_db.shards.0] @@ -133,6 +135,7 @@ sharding_function = "pg_bigint_hash" username = "simple_user" password = "simple_user" pool_size = 5 +statement_timeout = 30000 [pools.simple_db.shards.0] servers = [ diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 835bd10ea..645ff94bf 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -66,6 +66,18 @@ psql -U sharding_user -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_te # Replica/primary selection & more sharding tests psql -U sharding_user -e -h 127.0.0.1 -p 6432 -f tests/sharding/query_routing_test_primary_replica.sql > /dev/null +# Statement timeout tests +sed -i 's/statement_timeout = 0/statement_timeout = 100/' .circleci/pgcat.toml +kill -SIGHUP $(pgrep pgcat) # Reload config +sleep 0.2 + +# This should timeout +(! psql -U sharding_user -e -h 127.0.0.1 -p 6432 -c 'select pg_sleep(0.5)') + +# Disable statement timeout +sed -i 's/statement_timeout = 100/statement_timeout = 0/' .circleci/pgcat.toml +kill -SIGHUP $(pgrep pgcat) # Reload config again + # # ActiveRecord tests # diff --git a/pgcat.toml b/pgcat.toml index bc246f493..297611889 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -92,10 +92,14 @@ password = "sharding_user" # is the sum of pool_size across all users. pool_size = 9 +# Maximum query duration. Dangerous, but protetcts against DBs that died and a non-obvious way. +statement_timeout = 0 + [pools.sharded_db.users.1] username = "other_user" password = "other_user" pool_size = 21 +statement_timeout = 15000 # Shard 0 [pools.sharded_db.shards.0] @@ -133,6 +137,7 @@ sharding_function = "pg_bigint_hash" username = "simple_user" password = "simple_user" pool_size = 5 +statement_timeout = 0 [pools.simple_db.shards.0] servers = [ diff --git a/src/client.rs b/src/client.rs index 1dd1bccbc..b36eae09c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -499,7 +499,7 @@ where // The query router determines where the query is going to go, // e.g. primary, replica, which shard. let mut query_router = QueryRouter::new(); - let mut round_robin = 0; + let mut round_robin = rand::random(); // Our custom protocol loop. // We expect the client to either start a transaction with regular queries @@ -970,17 +970,54 @@ where } async fn receive_server_message( - &self, + &mut self, server: &mut Server, address: &Address, shard: usize, pool: &ConnectionPool, ) -> Result { - match server.recv().await { - Ok(message) => Ok(message), - Err(err) => { - pool.ban(address, shard, self.process_id); - Err(err) + if pool.settings.user.statement_timeout > 0 { + match tokio::time::timeout( + tokio::time::Duration::from_millis(pool.settings.user.statement_timeout), + server.recv(), + ) + .await + { + Ok(result) => match result { + Ok(message) => Ok(message), + Err(err) => { + pool.ban(address, shard, self.process_id); + error_response_terminal( + &mut self.write, + &format!("error receiving data from server: {:?}", err), + ) + .await?; + Err(err) + } + }, + Err(_) => { + error!( + "Statement timeout while talking to {:?} with user {}", + address, pool.settings.user.username + ); + server.mark_bad(); + pool.ban(address, shard, self.process_id); + error_response_terminal(&mut self.write, "pool statement timeout").await?; + Err(Error::StatementTimeout) + } + } + } else { + match server.recv().await { + Ok(message) => Ok(message), + Err(err) => { + pool.ban(address, shard, self.process_id); + error_response_terminal( + &mut self.write, + &format!("error receiving data from server: {:?}", err), + ) + .await?; + Err(err) + } } } } diff --git a/src/config.rs b/src/config.rs index 57b52ae18..ae006b39d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -100,6 +100,7 @@ pub struct User { pub username: String, pub password: String, pub pool_size: u32, + pub statement_timeout: u64, } impl Default for User { @@ -108,6 +109,7 @@ impl Default for User { username: String::from("postgres"), password: String::new(), pool_size: 15, + statement_timeout: 0, } } } @@ -332,6 +334,7 @@ impl Config { }; for (pool_name, pool_config) in &self.pools { + // TODO: Make this output prettier (maybe a table?) info!("--- Settings for pool {} ---", pool_name); info!( "Pool size from all users: {}", @@ -346,8 +349,17 @@ impl Config { info!("Sharding function: {}", pool_config.sharding_function); info!("Primary reads: {}", pool_config.primary_reads_enabled); info!("Query router: {}", pool_config.query_parser_enabled); + + // TODO: Make this prettier. info!("Number of shards: {}", pool_config.shards.len()); info!("Number of users: {}", pool_config.users.len()); + + for user in &pool_config.users { + info!( + "{} pool size: {}, statement timeout: {}", + user.1.username, user.1.pool_size, user.1.statement_timeout + ); + } } } } diff --git a/src/errors.rs b/src/errors.rs index cc8f65d00..06371fd19 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -11,4 +11,5 @@ pub enum Error { AllServersDown, ClientError, TlsError, + StatementTimeout, } From 4f0e60cb365ecb2b474b545179fa7a0761018aff Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 15 Aug 2022 22:51:37 -0500 Subject: [PATCH 181/551] Add cl_idle to SHOW POOLS (#124) --- src/admin.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/admin.rs b/src/admin.rs index 5c820ee7f..b82b29401 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -174,6 +174,7 @@ where let columns = vec![ ("database", DataType::Text), ("user", DataType::Text), + ("cl_idle", DataType::Numeric), ("cl_active", DataType::Numeric), ("cl_waiting", DataType::Numeric), ("cl_cancel_req", DataType::Numeric), From 2746dea43df43d8cead1db48666e4f87843146d4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 15 Aug 2022 23:54:49 -0700 Subject: [PATCH 182/551] Fix lost statistics (#125) * Lost events * more logging --- src/main.rs | 2 +- src/stats.rs | 52 ++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/main.rs b/src/main.rs index db934747b..0b2e1d59f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -133,7 +133,7 @@ async fn main() { let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); // Statistics reporting. - let (tx, rx) = mpsc::channel(100); + let (tx, rx) = mpsc::channel(100_000); REPORTER.store(Arc::new(Reporter::new(tx.clone()))); // Connection pool that allows to query all shards and replicas. diff --git a/src/stats.rs b/src/stats.rs index 83aa9cb5b..fde4071b8 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,9 +1,10 @@ use arc_swap::ArcSwap; /// Statistics and reporting. -use log::info; +use log::{error, info, trace}; use once_cell::sync::Lazy; use parking_lot::Mutex; use std::collections::HashMap; +use tokio::sync::mpsc::error::TrySendError; use tokio::sync::mpsc::{channel, Receiver, Sender}; use crate::pool::get_number_of_addresses; @@ -43,7 +44,7 @@ enum EventName { /// Event data sent to the collector /// from clients and servers. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Event { /// The name of the event being reported. name: EventName, @@ -79,6 +80,25 @@ impl Reporter { Reporter { tx: tx } } + /// Send statistics to the task keeping track of stats. + fn send(&self, event: Event) { + let name = event.name; + let result = self.tx.try_send(event); + + match result { + Ok(_) => trace!( + "{:?} event reported successfully, capacity: {}", + name, + self.tx.capacity() + ), + + Err(err) => match err { + TrySendError::Full { .. } => error!("{:?} event dropped, buffer full", name), + TrySendError::Closed { .. } => error!("{:?} event dropped, channel closed", name), + }, + }; + } + /// Report a query executed by a client against /// a server identified by the `address_id`. pub fn query(&self, process_id: i32, address_id: usize) { @@ -89,7 +109,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event); } /// Report a transaction executed by a client against @@ -102,7 +122,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Report data sent to a server identified by `address_id`. @@ -115,7 +135,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Report data received from a server identified by `address_id`. @@ -128,7 +148,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Time spent waiting to get a healthy connection from the pool @@ -142,7 +162,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Reports a client identified by `process_id` waiting for a connection @@ -155,7 +175,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Reports a client identified by `process_id` is done waiting for a connection @@ -168,7 +188,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Reports a client identified by `process_id` is done querying the server @@ -181,7 +201,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Reports a client identified by `process_id` is disconecting from the pooler. @@ -194,7 +214,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Reports a server connection identified by `process_id` for @@ -208,7 +228,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Reports a server connection identified by `process_id` for @@ -222,7 +242,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Reports a server connection identified by `process_id` for @@ -236,7 +256,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Reports a server connection identified by `process_id` for @@ -250,7 +270,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } /// Reports a server connection identified by `process_id` is disconecting from the pooler. @@ -263,7 +283,7 @@ impl Reporter { address_id: address_id, }; - let _ = self.tx.try_send(event); + self.send(event) } } From 251ffe982f3c4c9e9f669e1c2f0a6a5a661aa962 Mon Sep 17 00:00:00 2001 From: Pradeep Chhetri <30620077+chhetripradeep@users.noreply.github.com> Date: Wed, 17 Aug 2022 01:01:46 +0800 Subject: [PATCH 183/551] Minor cleanup in admin command (#126) * Minor cleanup in admin command * Typo correction * fix when the admin query is ending with semicolon --- examples/docker/pgcat.toml | 5 +++ pgcat.toml | 2 +- src/admin.rs | 65 +++++++++++++++++++++++--------------- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index 3c74df33c..85a3f466d 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -92,10 +92,14 @@ password = "postgres" # is the sum of pool_size across all users. pool_size = 9 +# Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. +statement_timeout = 0 + [pools.sharded.users.1] username = "postgres" password = "postgres" pool_size = 21 +statement_timeout = 15000 # Shard 0 [pools.sharded.shards.0] @@ -133,6 +137,7 @@ sharding_function = "pg_bigint_hash" username = "postgres" password = "postgres" pool_size = 5 +statement_timeout = 0 [pools.simple_db.shards.0] servers = [ diff --git a/pgcat.toml b/pgcat.toml index 297611889..9125afd5c 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -92,7 +92,7 @@ password = "sharding_user" # is the sum of pool_size across all users. pool_size = 9 -# Maximum query duration. Dangerous, but protetcts against DBs that died and a non-obvious way. +# Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. statement_timeout = 0 [pools.sharded_db.users.1] diff --git a/src/admin.rs b/src/admin.rs index b82b29401..6a79e49ee 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -44,32 +44,45 @@ where trace!("Admin query: {}", query); - if query.starts_with("SHOW STATS") { - trace!("SHOW STATS"); - show_stats(stream).await - } else if query.starts_with("RELOAD") { - trace!("RELOAD"); - reload(stream, client_server_map).await - } else if query.starts_with("SHOW CONFIG") { - trace!("SHOW CONFIG"); - show_config(stream).await - } else if query.starts_with("SHOW DATABASES") { - trace!("SHOW DATABASES"); - show_databases(stream).await - } else if query.starts_with("SHOW POOLS") { - trace!("SHOW POOLS"); - show_pools(stream).await - } else if query.starts_with("SHOW LISTS") { - trace!("SHOW LISTS"); - show_lists(stream).await - } else if query.starts_with("SHOW VERSION") { - trace!("SHOW VERSION"); - show_version(stream).await - } else if query.starts_with("SET ") { - trace!("SET"); - ignore_set(stream).await - } else { - error_response(stream, "Unsupported query against the admin database").await + let query_parts: Vec<&str> = query.trim_end_matches(';').split_whitespace().collect(); + + match query_parts[0] { + "RELOAD" => { + trace!("RELOAD"); + reload(stream, client_server_map).await + } + "SET" => { + trace!("SET"); + ignore_set(stream).await + } + "SHOW" => match query_parts[1] { + "CONFIG" => { + trace!("SHOW CONFIG"); + show_config(stream).await + } + "DATABASES" => { + trace!("SHOW DATABASES"); + show_databases(stream).await + } + "LISTS" => { + trace!("SHOW LISTS"); + show_lists(stream).await + } + "POOLS" => { + trace!("SHOW POOLS"); + show_pools(stream).await + } + "STATS" => { + trace!("SHOW STATS"); + show_stats(stream).await + } + "VERSION" => { + trace!("SHOW VERSION"); + show_version(stream).await + } + _ => error_response(stream, "Unsupported SHOW query against the admin database").await, + }, + _ => error_response(stream, "Unsupported query against the admin database").await, } } From f61b67ea677e4534a3cb94f8deb0e5d1c84cb157 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 17 Aug 2022 10:40:47 -0500 Subject: [PATCH 184/551] Add pool name and username to address object (#128) * Add pool name and username to address object * Fix address name * fmt --- src/config.rs | 8 ++++++-- src/pool.rs | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/config.rs b/src/config.rs index ae006b39d..ed3381045 100644 --- a/src/config.rs +++ b/src/config.rs @@ -64,6 +64,8 @@ pub struct Address { pub database: String, pub role: Role, pub replica_number: usize, + pub username: String, + pub poolname: String, } impl Default for Address { @@ -76,6 +78,8 @@ impl Default for Address { replica_number: 0, database: String::from("database"), role: Role::Replica, + username: String::from("username"), + poolname: String::from("poolname"), } } } @@ -84,11 +88,11 @@ impl Address { /// Address name (aka database) used in `SHOW STATS`, `SHOW DATABASES`, and `SHOW POOLS`. pub fn name(&self) -> String { match self.role { - Role::Primary => format!("{}_shard_{}_primary", self.database, self.shard), + Role::Primary => format!("{}_shard_{}_primary", self.poolname, self.shard), Role::Replica => format!( "{}_shard_{}_replica_{}", - self.database, self.shard, self.replica_number + self.poolname, self.shard, self.replica_number ), } } diff --git a/src/pool.rs b/src/pool.rs index 76445c8fb..4a700782f 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -114,12 +114,14 @@ impl ConnectionPool { let address = Address { id: address_id, - database: pool_name.clone(), + database: shard.database.clone(), host: server.0.clone(), port: server.1.to_string(), role: role, replica_number, shard: shard_idx.parse::().unwrap(), + username: user_info.username.clone(), + poolname: pool_name.clone(), }; address_id += 1; From 3b98ca56c1e802efc4f9cb83ce09e760030376b2 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 18 Aug 2022 08:12:38 -0400 Subject: [PATCH 185/551] Minor Refactoring of re-used code and server stat reporting (#129) * Minor changes to stats reporting and recduce re-used code * fmt --- src/client.rs | 112 +++++++++++++++++++++----------------------------- src/pool.rs | 4 +- 2 files changed, 49 insertions(+), 67 deletions(-) diff --git a/src/client.rs b/src/client.rs index b36eae09c..278cda8cc 100644 --- a/src/client.rs +++ b/src/client.rs @@ -667,7 +667,6 @@ where .client_disconnecting(self.process_id, last_address_id); } self.stats.client_active(self.process_id, address.id); - self.stats.server_active(server.process_id(), address.id); self.last_address_id = Some(address.id); self.last_server_id = Some(server.process_id()); @@ -731,44 +730,16 @@ where 'Q' => { debug!("Sending query to server"); - self.send_server_message( - server, + self.send_and_receive_loop( + code, original, + server, &address, query_router.shard(), &pool, ) .await?; - // Read all data the server has to offer, which can be multiple messages - // buffered in 8196 bytes chunks. - loop { - let response = self - .receive_server_message( - server, - &address, - query_router.shard(), - &pool, - ) - .await?; - - // Send server reply to the client. - match write_all_half(&mut self.write, response).await { - Ok(_) => (), - Err(err) => { - server.mark_bad(); - return Err(err); - } - }; - - if !server.is_data_available() { - break; - } - } - - // Report query executed statistics. - self.stats.query(self.process_id, address.id); - if !server.in_transaction() { // Report transaction executed statistics. self.stats.transaction(self.process_id, address.id); @@ -776,7 +747,6 @@ where // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { - self.stats.server_idle(server.process_id(), address.id); break; } } @@ -830,9 +800,10 @@ where self.buffer.put(&original[..]); - self.send_server_message( - server, + self.send_and_receive_loop( + code, self.buffer.clone(), + server, &address, query_router.shard(), &pool, @@ -841,41 +812,12 @@ where self.buffer.clear(); - // Read all data the server has to offer, which can be multiple messages - // buffered in 8196 bytes chunks. - loop { - let response = self - .receive_server_message( - server, - &address, - query_router.shard(), - &pool, - ) - .await?; - - match write_all_half(&mut self.write, response).await { - Ok(_) => (), - Err(err) => { - server.mark_bad(); - return Err(err); - } - }; - - if !server.is_data_available() { - break; - } - } - - // Report query executed statistics. - self.stats.query(self.process_id, address.id); - if !server.in_transaction() { self.stats.transaction(self.process_id, address.id); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { - self.stats.server_idle(server.process_id(), address.id); break; } } @@ -925,7 +867,6 @@ where // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { - self.stats.server_idle(server.process_id(), address.id); break; } } @@ -941,6 +882,7 @@ where // The server is no longer bound to us, we can't cancel it's queries anymore. debug!("Releasing server back into the pool"); + self.stats.server_idle(server.process_id(), address.id); self.release(); self.stats.client_idle(self.process_id, address.id); } @@ -952,6 +894,46 @@ where guard.remove(&(self.process_id, self.secret_key)); } + async fn send_and_receive_loop( + &mut self, + code: char, + message: BytesMut, + server: &mut Server, + address: &Address, + shard: usize, + pool: &ConnectionPool, + ) -> Result<(), Error> { + debug!("Sending {} to server", code); + + self.send_server_message(server, message, &address, shard, &pool) + .await?; + + // Read all data the server has to offer, which can be multiple messages + // buffered in 8196 bytes chunks. + loop { + let response = self + .receive_server_message(server, &address, shard, &pool) + .await?; + + match write_all_half(&mut self.write, response).await { + Ok(_) => (), + Err(err) => { + server.mark_bad(); + return Err(err); + } + }; + + if !server.is_data_available() { + break; + } + } + + // Report query executed statistics. + self.stats.query(self.process_id, address.id); + + Ok(()) + } + async fn send_server_message( &self, server: &mut Server, diff --git a/src/pool.rs b/src/pool.rs index 4a700782f..5684d5452 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -335,7 +335,7 @@ impl ConnectionPool { if !require_healthcheck { self.stats .checkout_time(now.elapsed().as_micros(), process_id, address.id); - self.stats.server_idle(conn.process_id(), address.id); + self.stats.server_active(conn.process_id(), address.id); return Ok((conn, address.clone())); } @@ -354,7 +354,7 @@ impl ConnectionPool { Ok(_) => { self.stats .checkout_time(now.elapsed().as_micros(), process_id, address.id); - self.stats.server_idle(conn.process_id(), address.id); + self.stats.server_active(conn.process_id(), address.id); return Ok((conn, address.clone())); } From 5788afcb41409d4c9c61cc03dfc5ee3e13b413ab Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 22 Aug 2022 00:15:20 -0500 Subject: [PATCH 186/551] Random instance selection (#136) * wip * revert some' * revert more * poor-man's integration test * remove test * fmt * --workspace * fix build * fix integration test * another stab * log * run after integration * cargo test after integration * revert * revert more * Refactor + clean up * more clean up --- src/client.rs | 10 +---- src/config.rs | 6 +-- src/pool.rs | 113 +++++++++++++++++++------------------------------- 3 files changed, 46 insertions(+), 83 deletions(-) diff --git a/src/client.rs b/src/client.rs index 278cda8cc..0c553f83b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -499,7 +499,6 @@ where // The query router determines where the query is going to go, // e.g. primary, replica, which shard. let mut query_router = QueryRouter::new(); - let mut round_robin = rand::random(); // Our custom protocol loop. // We expect the client to either start a transaction with regular queries @@ -631,12 +630,7 @@ where // Grab a server from the pool. let connection = match pool - .get( - query_router.shard(), - query_router.role(), - self.process_id, - round_robin, - ) + .get(query_router.shard(), query_router.role(), self.process_id) .await { Ok(conn) => { @@ -655,8 +649,6 @@ where let address = connection.1; let server = &mut *reference; - round_robin += 1; - // Server is assigned to the client in case the client wants to // cancel a query later. server.claim(self.process_id, self.secret_key); diff --git a/src/config.rs b/src/config.rs index ed3381045..9d1658ffe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -63,7 +63,7 @@ pub struct Address { pub shard: usize, pub database: String, pub role: Role, - pub replica_number: usize, + pub instance_index: usize, pub username: String, pub poolname: String, } @@ -75,7 +75,7 @@ impl Default for Address { host: String::from("127.0.0.1"), port: String::from("5432"), shard: 0, - replica_number: 0, + instance_index: 0, database: String::from("database"), role: Role::Replica, username: String::from("username"), @@ -92,7 +92,7 @@ impl Address { Role::Replica => format!( "{}_shard_{}_replica_{}", - self.poolname, self.shard, self.replica_number + self.poolname, self.shard, self.instance_index ), } } diff --git a/src/pool.rs b/src/pool.rs index 5684d5452..cbb9b43f9 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -6,6 +6,8 @@ use chrono::naive::NaiveDateTime; use log::{debug, error, info, warn}; use once_cell::sync::Lazy; use parking_lot::{Mutex, RwLock}; +use rand::seq::SliceRandom; +use rand::thread_rng; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; @@ -118,7 +120,7 @@ impl ConnectionPool { host: server.0.clone(), port: server.1.to_string(), role: role, - replica_number, + instance_index: replica_number, shard: shard_idx.parse::().unwrap(), username: user_info.username.clone(), poolname: pool_name.clone(), @@ -201,16 +203,9 @@ impl ConnectionPool { /// the pooler starts up. async fn validate(&mut self) -> Result<(), Error> { let mut server_infos = Vec::new(); - let stats = self.stats.clone(); - for shard in 0..self.shards() { - let mut round_robin = 0; - - for _ in 0..self.servers(shard) { - // To keep stats consistent. - let fake_process_id = 0; - - let connection = match self.get(shard, None, fake_process_id, round_robin).await { + for index in 0..self.servers(shard) { + let connection = match self.databases[shard][index].get().await { Ok(conn) => conn, Err(err) => { error!("Shard {} down or misconfigured: {:?}", shard, err); @@ -218,25 +213,20 @@ impl ConnectionPool { } }; - let proxy = connection.0; - let address = connection.1; + let proxy = connection; let server = &*proxy; let server_info = server.server_info(); - stats.client_disconnecting(fake_process_id, address.id); - if server_infos.len() > 0 { // Compare against the last server checked. if server_info != server_infos[server_infos.len() - 1] { warn!( "{:?} has different server configuration than the last server", - address + proxy.address() ); } } - server_infos.push(server_info); - round_robin += 1; } } @@ -254,58 +244,31 @@ impl ConnectionPool { /// Get a connection from the pool. pub async fn get( &self, - shard: usize, // shard number - role: Option, // primary or replica - process_id: i32, // client id - mut round_robin: usize, // round robin offset + shard: usize, // shard number + role: Option, // primary or replica + process_id: i32, // client id ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { let now = Instant::now(); - let addresses = &self.addresses[shard]; - - let mut allowed_attempts = match role { - // Primary-specific queries get one attempt, if the primary is down, - // nothing we should do about it I think. It's dangerous to retry - // write queries. - Some(Role::Primary) => 1, + let mut candidates: Vec
= self.addresses[shard] + .clone() + .into_iter() + .filter(|address| address.role == role) + .collect(); - // Replicas get to try as many times as there are replicas - // and connections in the pool. - _ => addresses.len(), - }; - - debug!("Allowed attempts for {:?}: {}", role, allowed_attempts); - - let exists = match role { - Some(role) => addresses.iter().filter(|addr| addr.role == role).count() > 0, - None => true, - }; - - if !exists { - error!("Requested role {:?}, but none are configured", role); - return Err(Error::BadConfig); - } + // Random load balancing + candidates.shuffle(&mut thread_rng()); let healthcheck_timeout = get_config().general.healthcheck_timeout; let healthcheck_delay = get_config().general.healthcheck_delay as u128; - while allowed_attempts > 0 { - // Round-robin replicas. - round_robin += 1; - - let index = round_robin % addresses.len(); - let address = &addresses[index]; - - // Make sure you're getting a primary or a replica - // as per request. If no specific role is requested, the first - // available will be chosen. - if address.role != role { - continue; - } - - allowed_attempts -= 1; + while !candidates.is_empty() { + // Get the next candidate + let address = match candidates.pop() { + Some(address) => address, + None => break, + }; - // Don't attempt to connect to banned servers. - if self.is_banned(address, shard, role) { + if self.is_banned(&address, address.shard, role) { continue; } @@ -313,11 +276,14 @@ impl ConnectionPool { self.stats.client_waiting(process_id, address.id); // Check if we can connect - let mut conn = match self.databases[shard][index].get().await { + let mut conn = match self.databases[address.shard][address.instance_index] + .get() + .await + { Ok(conn) => conn, Err(err) => { - error!("Banning replica {}, error: {:?}", index, err); - self.ban(address, shard, process_id); + error!("Banning instance {:?}, error: {:?}", address, err); + self.ban(&address, address.shard, process_id); self.stats.client_disconnecting(process_id, address.id); self.stats .checkout_time(now.elapsed().as_micros(), process_id, address.id); @@ -359,29 +325,34 @@ impl ConnectionPool { } // Health check failed. - Err(_) => { - error!("Banning replica {} because of failed health check", index); + Err(err) => { + error!( + "Banning instance {:?} because of failed health check, {:?}", + address, err + ); // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(address, shard, process_id); + self.ban(&address, address.shard, process_id); continue; } }, // Health check timed out. - Err(_) => { - error!("Banning replica {} because of health check timeout", index); + Err(err) => { + error!( + "Banning instance {:?} because of health check timeout, {:?}", + address, err + ); // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(address, shard, process_id); + self.ban(&address, address.shard, process_id); continue; } } } - return Err(Error::AllServersDown); } From d4efc7af93286b9af3242b0e9ea45174ba30a658 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 21 Aug 2022 22:20:31 -0700 Subject: [PATCH 187/551] Random lb (#138) --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 90cbd823a..48f835348 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover su | Session pooling | :white_check_mark: | Identical to PgBouncer. | | `COPY` support | :white_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | | Query cancellation | :white_check_mark: | Supported both in transaction and session pooling modes. | -| Load balancing of read queries | :white_check_mark: | Using round-robin between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | +| Load balancing of read queries | :white_check_mark: | Using random between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | | Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | | Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | | Statistics | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. | @@ -63,7 +63,7 @@ psql -h 127.0.0.1 -p 6432 -c 'SELECT 1' | `database` | The name of the database to connect to. This is the same on all servers that are part of one shard. | | | | | | | **`query_router`** | | | -| `default_role` | Traffic is routed to this role by default (round-robin), unless the client specifies otherwise. Default is `any`, for any role available. | `any`, `primary`, `replica` | +| `default_role` | Traffic is routed to this role by default (random), unless the client specifies otherwise. Default is `any`, for any role available. | `any`, `primary`, `replica` | | `query_parser_enabled` | Enable the query parser which will inspect incoming queries and route them to a primary or replicas. | `false` | | `primary_reads_enabled` | Enable this to allow read queries on the primary; otherwise read queries are routed to the replicas. | `true` | @@ -112,7 +112,7 @@ In transaction mode, a client talks to one server for the duration of a single t This mode is enabled by default. ### Load balancing of read queries -All queries are load balanced against the configured servers using the round-robin algorithm. The most straight forward configuration example would be to put this pooler in front of several replicas and let it load balance all queries. +All queries are load balanced against the configured servers using the random algorithm. The most straight forward configuration example would be to put this pooler in front of several replicas and let it load balance all queries. If the configuration includes a primary and replicas, the queries can be separated with the built-in query parser. The query parser will interpret the query and route all `SELECT` queries to a replica, while all other queries including explicit transactions will be routed to the primary. @@ -151,18 +151,18 @@ Failover behavior can get pretty interesting (read complex) when multiple config | **Query** | **`SET SERVER ROLE TO`** | **`query_parser_enabled`** | **`primary_reads_enabled`** | **Target state** | **Outcome** | |---------------------------|--------------------------|----------------------------|-----------------------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Read query, i.e. `SELECT` | unset (any) | false | false | up | Query is routed to the first instance in the round-robin loop. | -| Read query | unset (any) | true | false | up | Query is routed to the first replica instance in the round-robin loop. | -| Read query | unset (any) | true | true | up | Query is routed to the first instance in the round-robin loop. | -| Read query | replica | false | false | up | Query is routed to the first replica instance in the round-robin loop. | +| Read query, i.e. `SELECT` | unset (any) | false | false | up | Query is routed to the first instance in the random loop. | +| Read query | unset (any) | true | false | up | Query is routed to the first replica instance in the random loop. | +| Read query | unset (any) | true | true | up | Query is routed to the first instance in the random loop. | +| Read query | replica | false | false | up | Query is routed to the first replica instance in the random loop. | | Read query | primary | false | false | up | Query is routed to the primary. | -| Read query | unset (any) | false | false | down | First instance is banned for reads. Next target in the round-robin loop is attempted. | -| Read query | unset (any) | true | false | down | First replica instance is banned. Next replica instance is attempted in the round-robin loop. | -| Read query | unset (any) | true | true | down | First instance (even if primary) is banned for reads. Next instance is attempted in the round-robin loop. | -| Read query | replica | false | false | down | First replica instance is banned. Next replica instance is attempted in the round-robin loop. | +| Read query | unset (any) | false | false | down | First instance is banned for reads. Next target in the random loop is attempted. | +| Read query | unset (any) | true | false | down | First replica instance is banned. Next replica instance is attempted in the random loop. | +| Read query | unset (any) | true | true | down | First instance (even if primary) is banned for reads. Next instance is attempted in the random loop. | +| Read query | replica | false | false | down | First replica instance is banned. Next replica instance is attempted in the random loop. | | Read query | primary | false | false | down | The query is attempted against the primary and fails. The client receives an error. | | | | | | | | -| Write query e.g. `INSERT` | unset (any) | false | false | up | The query is attempted against the first available instance in the round-robin loop. If the instance is a replica, the query fails and the client receives an error. | +| Write query e.g. `INSERT` | unset (any) | false | false | up | The query is attempted against the first available instance in the random loop. If the instance is a replica, the query fails and the client receives an error. | | Write query | unset (any) | true | false | up | The query is routed to the primary. | | Write query | unset (any) | true | true | up | The query is routed to the primary. | | Write query | primary | false | false | up | The query is routed to the primary. | From be25d8d93884f4989eae3fef09de111c270d2fef Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 21 Aug 2022 22:40:49 -0700 Subject: [PATCH 188/551] Fix incorrect routing for replicas (#139) * Fix incorrect routing for replicas * name --- src/config.rs | 14 ++++++++------ src/pool.rs | 9 ++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/config.rs b/src/config.rs index 9d1658ffe..b75169373 100644 --- a/src/config.rs +++ b/src/config.rs @@ -63,9 +63,10 @@ pub struct Address { pub shard: usize, pub database: String, pub role: Role, - pub instance_index: usize, + pub replica_number: usize, + pub address_index: usize, pub username: String, - pub poolname: String, + pub pool_name: String, } impl Default for Address { @@ -75,11 +76,12 @@ impl Default for Address { host: String::from("127.0.0.1"), port: String::from("5432"), shard: 0, - instance_index: 0, + address_index: 0, + replica_number: 0, database: String::from("database"), role: Role::Replica, username: String::from("username"), - poolname: String::from("poolname"), + pool_name: String::from("pool_name"), } } } @@ -88,11 +90,11 @@ impl Address { /// Address name (aka database) used in `SHOW STATS`, `SHOW DATABASES`, and `SHOW POOLS`. pub fn name(&self) -> String { match self.role { - Role::Primary => format!("{}_shard_{}_primary", self.poolname, self.shard), + Role::Primary => format!("{}_shard_{}_primary", self.pool_name, self.shard), Role::Replica => format!( "{}_shard_{}_replica_{}", - self.poolname, self.shard, self.instance_index + self.pool_name, self.shard, self.replica_number ), } } diff --git a/src/pool.rs b/src/pool.rs index cbb9b43f9..ac5bc9110 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -102,6 +102,7 @@ impl ConnectionPool { let shard = &pool_config.shards[&shard_idx]; let mut pools = Vec::new(); let mut servers = Vec::new(); + let mut address_index = 0; let mut replica_number = 0; for server in shard.servers.iter() { @@ -120,13 +121,15 @@ impl ConnectionPool { host: server.0.clone(), port: server.1.to_string(), role: role, - instance_index: replica_number, + address_index, + replica_number, shard: shard_idx.parse::().unwrap(), username: user_info.username.clone(), - poolname: pool_name.clone(), + pool_name: pool_name.clone(), }; address_id += 1; + address_index += 1; if role == Role::Replica { replica_number += 1; @@ -276,7 +279,7 @@ impl ConnectionPool { self.stats.client_waiting(process_id, address.id); // Check if we can connect - let mut conn = match self.databases[address.shard][address.instance_index] + let mut conn = match self.databases[address.shard][address.address_index] .get() .await { From f51e0a0389695c5c815b97e3b733d62c13b0d208 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 22 Aug 2022 11:52:34 -0700 Subject: [PATCH 189/551] Fix too many idle servers (#140) * Fix too many idle servers * oops --- src/client.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/client.rs b/src/client.rs index 0c553f83b..c2ad87dde 100644 --- a/src/client.rs +++ b/src/client.rs @@ -73,6 +73,9 @@ pub struct Client { /// Last server process id we talked to. last_server_id: Option, + /// Connected to server + connected_to_server: bool, + /// Name of the server pool for this client (This comes from the database name in the connection string) target_pool_name: String, @@ -429,6 +432,7 @@ where target_pool_name: target_pool_name.clone(), target_user_name: target_user_name.clone(), shutdown_event_receiver: shutdown_event_receiver, + connected_to_server: false, }); } @@ -461,6 +465,7 @@ where target_pool_name: String::from("undefined"), target_user_name: String::from("undefined"), shutdown_event_receiver: shutdown_event_receiver, + connected_to_server: false, }); } @@ -652,6 +657,7 @@ where // Server is assigned to the client in case the client wants to // cancel a query later. server.claim(self.process_id, self.secret_key); + self.connected_to_server = true; // Update statistics. if let Some(last_address_id) = self.last_address_id { @@ -875,6 +881,7 @@ where // The server is no longer bound to us, we can't cancel it's queries anymore. debug!("Releasing server back into the pool"); self.stats.server_idle(server.process_id(), address.id); + self.connected_to_server = false; self.release(); self.stats.client_idle(self.process_id, address.id); } @@ -1002,15 +1009,16 @@ impl Drop for Client { let mut guard = self.client_server_map.lock(); guard.remove(&(self.process_id, self.secret_key)); - // Update statistics. - if let Some(address_id) = self.last_address_id { - self.stats.client_disconnecting(self.process_id, address_id); + // Dirty shutdown + // TODO: refactor, this is not the best way to handle state management. + if self.connected_to_server { + if let Some(address_id) = self.last_address_id { + self.stats.client_disconnecting(self.process_id, address_id); - if let Some(process_id) = self.last_server_id { - self.stats.server_idle(process_id, address_id); + if let Some(process_id) = self.last_server_id { + self.stats.server_idle(process_id, address_id); + } } } - - // self.release(); } } From 105325202148c06086629da9a8474169128a9885 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 22 Aug 2022 11:56:40 -0700 Subject: [PATCH 190/551] Really fix idle servers (#141) --- src/client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client.rs b/src/client.rs index c2ad87dde..c4866a089 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1011,10 +1011,10 @@ impl Drop for Client { // Dirty shutdown // TODO: refactor, this is not the best way to handle state management. - if self.connected_to_server { - if let Some(address_id) = self.last_address_id { - self.stats.client_disconnecting(self.process_id, address_id); + if let Some(address_id) = self.last_address_id { + self.stats.client_disconnecting(self.process_id, address_id); + if self.connected_to_server { if let Some(process_id) = self.last_server_id { self.stats.server_idle(process_id, address_id); } From 13fc8ccb600a13fa0b27603709e84f12351acbad Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 23 Aug 2022 13:02:23 -0500 Subject: [PATCH 191/551] Avoid sending `Z` packet in the middle of extended protocol packet sequence if we fail to get connection from pool (#137) * Failing test * maybe * try fail * try * add message * pool size * correct user * more * debug * try fix * see stdout * stick? * fix configs * modify * types * m * maybe * make tests idempotent * hopefully fails * Add client fix * revert pgcat.toml change * Fix tests --- .circleci/run_tests.sh | 9 ++-- src/client.rs | 17 +++++-- tests/ruby/tests.rb | 110 ++++++++++++++++++++++++++++++++--------- 3 files changed, 106 insertions(+), 30 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 645ff94bf..4202eb5ed 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -19,8 +19,8 @@ PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard2 -i # Install Toxiproxy to simulate a downed/slow database -wget -O toxiproxy-2.1.4.deb https://github.com/Shopify/toxiproxy/releases/download/v2.1.4/toxiproxy_2.1.4_amd64.deb -sudo dpkg -i toxiproxy-2.1.4.deb +wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb +sudo dpkg -i toxiproxy-2.4.0.deb # Start Toxiproxy toxiproxy-server & @@ -129,11 +129,14 @@ toxiproxy-cli toxic remove --toxicName latency_downstream postgres_replica start_pgcat "info" # Test session mode (and config reload) -sed -i 's/pool_mode = "transaction"/pool_mode = "session"/' .circleci/pgcat.toml +sed -i '0,/simple_db/s/pool_mode = "transaction"/pool_mode = "session"/' .circleci/pgcat.toml # Reload config test kill -SIGHUP $(pgrep pgcat) +# Revert settings after reload. Makes test runs idempotent +sed -i '0,/simple_db/s/pool_mode = "session"/pool_mode = "transaction"/' .circleci/pgcat.toml + sleep 1 # Prepared statements that will only work in session mode diff --git a/src/client.rs b/src/client.rs index c4866a089..0612c27f9 100644 --- a/src/client.rs +++ b/src/client.rs @@ -643,9 +643,20 @@ where conn } Err(err) => { - error!("Could not get connection from pool: {:?}", err); - error_response(&mut self.write, "could not get connection from the pool") - .await?; + // Clients do not expect to get SystemError followed by ReadyForQuery in the middle + // of extended protocol submission. So we will hold off on sending the actual error + // message to the client until we get 'S' message + match message[0] as char { + 'P' | 'B' | 'E' | 'D' => (), + _ => { + error!("Could not get connection from pool: {:?}", err); + error_response( + &mut self.write, + "could not get connection from the pool", + ) + .await?; + } + } continue; } }; diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index ba9476f43..d7b928d4a 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -5,6 +5,89 @@ require 'toml' $stdout.sync = true +$stderr.sync = true + +class ConfigEditor + def initialize + @original_config_text = File.read('../../.circleci/pgcat.toml') + text_to_load = @original_config_text.gsub("5432", "\"5432\"") + + @original_configs = TOML.load(text_to_load) + end + + def original_configs + TOML.load(TOML::Generator.new(@original_configs).body) + end + + def with_modified_configs(new_configs) + text_to_write = TOML::Generator.new(new_configs).body + text_to_write = text_to_write.gsub("\"5432\"", "5432") + File.write('../../.circleci/pgcat.toml', text_to_write) + yield + ensure + File.write('../../.circleci/pgcat.toml', @original_config_text) + end +end + +def with_captured_stdout_stderr + sout = STDOUT.clone + serr = STDERR.clone + STDOUT.reopen("/tmp/out.txt", "w+") + STDERR.reopen("/tmp/err.txt", "w+") + STDOUT.sync = true + STDERR.sync = true + yield + return File.read('/tmp/out.txt'), File.read('/tmp/err.txt') +ensure + STDOUT.reopen(sout) + STDERR.reopen(serr) +end + + +def test_extended_protocol_pooler_errors + admin_conn = PG::connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat") + + conf_editor = ConfigEditor.new + new_configs = conf_editor.original_configs + + # shorter timeouts + new_configs["general"]["connect_timeout"] = 500 + new_configs["general"]["ban_time"] = 1 + new_configs["general"]["shutdown_timeout"] = 1 + new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 + new_configs["pools"]["sharded_db"]["users"]["1"]["pool_size"] = 1 + + conf_editor.with_modified_configs(new_configs) { admin_conn.async_exec("RELOAD") } + + conn_str = "postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db" + 10.times do + Thread.new do + conn = PG::connect(conn_str) + conn.async_exec("SELECT pg_sleep(5)") rescue PG::SystemError + ensure + conn&.close + end + end + + sleep(0.5) + conn_under_test = PG::connect(conn_str) + stdout, stderr = with_captured_stdout_stderr do + 5.times do |i| + conn_under_test.async_exec("SELECT 1") rescue PG::SystemError + conn_under_test.exec_params("SELECT #{i} + $1", [i]) rescue PG::SystemError + sleep 1 + end + end + + raise StandardError, "Libpq got unexpected messages while idle" if stderr.include?("arrived from server while idle") + puts "Pool checkout errors not breaking clients passed" +ensure + sleep 1 + admin_conn.async_exec("RELOAD") # Reset state + conn_under_test&.close +end + +test_extended_protocol_pooler_errors # Uncomment these two to see all queries. # ActiveRecord.verbose_query_logs = true @@ -144,30 +227,6 @@ def test_server_parameters end -class ConfigEditor - def initialize - @original_config_text = File.read('../../.circleci/pgcat.toml') - text_to_load = @original_config_text.gsub("5432", "\"5432\"") - - @original_configs = TOML.load(text_to_load) - end - - def original_configs - TOML.load(TOML::Generator.new(@original_configs).body) - end - - def with_modified_configs(new_configs) - text_to_write = TOML::Generator.new(new_configs).body - text_to_write = text_to_write.gsub("\"5432\"", "5432") - File.write('../../.circleci/pgcat.toml', text_to_write) - yield - ensure - File.write('../../.circleci/pgcat.toml', @original_config_text) - end - -end - - def test_reload_pool_recycling admin_conn = PG::connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat") server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") @@ -201,3 +260,6 @@ def test_reload_pool_recycling end test_reload_pool_recycling + + + From d87e115761b3e2be02de59f4d8be27bbb4ecf617 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 25 Aug 2022 06:40:56 -0700 Subject: [PATCH 192/551] Graceful shutdown and refactor (#144) * Graceful shutdown and refactor * ok * _Graceful_ shutdown * Remove hardcoded setting * clean up * end * timeout * hmm * hmm! * bash * bash * hmm * maybe maybe * Adds tests and move non-admin connection rejection to startup (#145) * Move error response * Adds tests and removes unused variable * Adds debug log --- src/admin.rs | 4 +- src/client.rs | 270 ++++++++++++++++++++++++------------------ src/config.rs | 78 +++++++++--- src/errors.rs | 1 + src/main.rs | 248 +++++++++++++++++++------------------- src/pool.rs | 158 ++++++++++++++++-------- src/query_router.rs | 54 ++++----- src/server.rs | 6 +- tests/python/tests.py | 137 +++++++++++++++++---- 9 files changed, 588 insertions(+), 368 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 6a79e49ee..4576d1681 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -265,11 +265,11 @@ where for (_, pool) in get_all_pools() { let pool_config = pool.settings.clone(); for shard in 0..pool.shards() { - let database_name = &pool_config.shards[&shard.to_string()].database; + let database_name = &pool.address(shard, 0).database; for server in 0..pool.servers(shard) { let address = pool.address(shard, server); let pool_state = pool.pool_state(shard, server); - let banned = pool.is_banned(address, shard, Some(address.role)); + let banned = pool.is_banned(address, Some(address.role)); res.put(data_row(&vec![ address.name(), // name diff --git a/src/client.rs b/src/client.rs index 0612c27f9..419448fb8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -5,13 +5,14 @@ use std::collections::HashMap; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; use tokio::net::TcpStream; use tokio::sync::broadcast::Receiver; +use tokio::sync::mpsc::Sender; use crate::admin::{generate_server_info_for_admin, handle_admin}; use crate::config::{get_config, Address}; use crate::constants::*; use crate::errors::Error; use crate::messages::*; -use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; +use crate::pool::{get_pool, ClientServerMap, ConnectionPool, PoolMode}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; use crate::stats::{get_reporter, Reporter}; @@ -58,7 +59,6 @@ pub struct Client { client_server_map: ClientServerMap, /// Client parameters, e.g. user, client_encoding, etc. - #[allow(dead_code)] parameters: HashMap, /// Statistics @@ -77,20 +77,22 @@ pub struct Client { connected_to_server: bool, /// Name of the server pool for this client (This comes from the database name in the connection string) - target_pool_name: String, + pool_name: String, /// Postgres user for this client (This comes from the user in the connection string) - target_user_name: String, + username: String, /// Used to notify clients about an impending shutdown - shutdown_event_receiver: Receiver<()>, + shutdown: Receiver<()>, } /// Client entrypoint. pub async fn client_entrypoint( mut stream: TcpStream, client_server_map: ClientServerMap, - shutdown_event_receiver: Receiver<()>, + shutdown: Receiver<()>, + drain: Sender, + admin_only: bool, ) -> Result<(), Error> { // Figure out if the client wants TLS or not. let addr = stream.peer_addr().unwrap(); @@ -109,11 +111,21 @@ pub async fn client_entrypoint( write_all(&mut stream, yes).await?; // Negotiate TLS. - match startup_tls(stream, client_server_map, shutdown_event_receiver).await { + match startup_tls(stream, client_server_map, shutdown, admin_only).await { Ok(mut client) => { info!("Client {:?} connected (TLS)", addr); - client.handle().await + if !client.is_admin() { + let _ = drain.send(1).await; + } + + let result = client.handle().await; + + if !client.is_admin() { + let _ = drain.send(-1).await; + } + + result } Err(err) => Err(err), } @@ -139,14 +151,25 @@ pub async fn client_entrypoint( addr, bytes, client_server_map, - shutdown_event_receiver, + shutdown, + admin_only, ) .await { Ok(mut client) => { info!("Client {:?} connected (plain)", addr); - client.handle().await + if !client.is_admin() { + let _ = drain.send(1).await; + } + + let result = client.handle().await; + + if !client.is_admin() { + let _ = drain.send(-1).await; + } + + result } Err(err) => Err(err), } @@ -169,14 +192,25 @@ pub async fn client_entrypoint( addr, bytes, client_server_map, - shutdown_event_receiver, + shutdown, + admin_only, ) .await { Ok(mut client) => { info!("Client {:?} connected (plain)", addr); - client.handle().await + if client.is_admin() { + let _ = drain.send(1).await; + } + + let result = client.handle().await; + + if !client.is_admin() { + let _ = drain.send(-1).await; + } + + result } Err(err) => Err(err), } @@ -187,20 +221,21 @@ pub async fn client_entrypoint( let (read, write) = split(stream); // Continue with cancel query request. - match Client::cancel( - read, - write, - addr, - bytes, - client_server_map, - shutdown_event_receiver, - ) - .await - { + match Client::cancel(read, write, addr, bytes, client_server_map, shutdown).await { Ok(mut client) => { info!("Client {:?} issued a cancel query request", addr); - client.handle().await + if client.is_admin() { + let _ = drain.send(1).await; + } + + let result = client.handle().await; + + if !client.is_admin() { + let _ = drain.send(-1).await; + } + + result } Err(err) => Err(err), @@ -253,7 +288,8 @@ where pub async fn startup_tls( stream: TcpStream, client_server_map: ClientServerMap, - shutdown_event_receiver: Receiver<()>, + shutdown: Receiver<()>, + admin_only: bool, ) -> Result>, WriteHalf>>, Error> { // Negotiate TLS. let tls = Tls::new()?; @@ -283,7 +319,8 @@ pub async fn startup_tls( addr, bytes, client_server_map, - shutdown_event_receiver, + shutdown, + admin_only, ) .await } @@ -298,6 +335,10 @@ where S: tokio::io::AsyncRead + std::marker::Unpin, T: tokio::io::AsyncWrite + std::marker::Unpin, { + pub fn is_admin(&self) -> bool { + self.admin + } + /// Handle Postgres client startup after TLS negotiation is complete /// or over plain text. pub async fn startup( @@ -306,29 +347,44 @@ where addr: std::net::SocketAddr, bytes: BytesMut, // The rest of the startup message. client_server_map: ClientServerMap, - shutdown_event_receiver: Receiver<()>, + shutdown: Receiver<()>, + admin_only: bool, ) -> Result, Error> { let config = get_config(); let stats = get_reporter(); - - trace!("Got StartupMessage"); let parameters = parse_startup(bytes.clone())?; - let target_pool_name = match parameters.get("database") { + + // These two parameters are mandatory by the protocol. + let pool_name = match parameters.get("database") { Some(db) => db, None => return Err(Error::ClientError), }; - let target_user_name = match parameters.get("user") { + let username = match parameters.get("user") { Some(user) => user, None => return Err(Error::ClientError), }; let admin = ["pgcat", "pgbouncer"] .iter() - .filter(|db| *db == &target_pool_name) + .filter(|db| *db == &pool_name) .count() == 1; + // Kick any client that's not admin while we're in admin-only mode. + if !admin && admin_only { + debug!( + "Rejecting non-admin connection to {} when in admin only mode", + pool_name + ); + error_response_terminal( + &mut write, + &format!("terminating connection due to administrator command"), + ) + .await?; + return Err(Error::ShuttingDown); + } + // Generate random backend ID and secret key let process_id: i32 = rand::random(); let secret_key: i32 = rand::random(); @@ -360,46 +416,55 @@ where Err(_) => return Err(Error::SocketError), }; + // Authenticate admin user. let (transaction_mode, server_info) = if admin { - let correct_user = config.general.admin_username.as_str(); - let correct_password = config.general.admin_password.as_str(); - // Compare server and client hashes. - let password_hash = md5_hash_password(correct_user, correct_password, &salt); + let password_hash = md5_hash_password( + &config.general.admin_username, + &config.general.admin_password, + &salt, + ); + if password_hash != password_response { debug!("Password authentication failed"); - wrong_password(&mut write, target_user_name).await?; + wrong_password(&mut write, username).await?; + return Err(Error::ClientError); } (false, generate_server_info_for_admin()) - } else { - let target_pool = match get_pool(target_pool_name.clone(), target_user_name.clone()) { + } + // Authenticate normal user. + else { + let pool = match get_pool(pool_name.clone(), username.clone()) { Some(pool) => pool, None => { error_response( &mut write, &format!( "No pool configured for database: {:?}, user: {:?}", - target_pool_name, target_user_name + pool_name, username ), ) .await?; + return Err(Error::ClientError); } }; - let transaction_mode = target_pool.settings.pool_mode == "transaction"; - let server_info = target_pool.server_info(); + // Compare server and client hashes. - let correct_password = target_pool.settings.user.password.as_str(); - let password_hash = md5_hash_password(&target_user_name, correct_password, &salt); + let password_hash = md5_hash_password(&username, &pool.settings.user.password, &salt); if password_hash != password_response { debug!("Password authentication failed"); - wrong_password(&mut write, &target_user_name).await?; + wrong_password(&mut write, username).await?; + return Err(Error::ClientError); } - (transaction_mode, server_info) + + let transaction_mode = pool.settings.pool_mode == PoolMode::Transaction; + + (transaction_mode, pool.server_info()) }; debug!("Password authentication successful"); @@ -411,27 +476,24 @@ where trace!("Startup OK"); - // Split the read and write streams - // so we can control buffering. - return Ok(Client { read: BufReader::new(read), write: write, addr, buffer: BytesMut::with_capacity(8196), cancel_mode: false, - transaction_mode: transaction_mode, - process_id: process_id, - secret_key: secret_key, - client_server_map: client_server_map, + transaction_mode, + process_id, + secret_key, + client_server_map, parameters: parameters.clone(), stats: stats, admin: admin, last_address_id: None, last_server_id: None, - target_pool_name: target_pool_name.clone(), - target_user_name: target_user_name.clone(), - shutdown_event_receiver: shutdown_event_receiver, + pool_name: pool_name.clone(), + username: username.clone(), + shutdown, connected_to_server: false, }); } @@ -443,7 +505,7 @@ where addr: std::net::SocketAddr, mut bytes: BytesMut, // The rest of the startup message. client_server_map: ClientServerMap, - shutdown_event_receiver: Receiver<()>, + shutdown: Receiver<()>, ) -> Result, Error> { let process_id = bytes.get_i32(); let secret_key = bytes.get_i32(); @@ -454,17 +516,17 @@ where buffer: BytesMut::with_capacity(8196), cancel_mode: true, transaction_mode: false, - process_id: process_id, - secret_key: secret_key, - client_server_map: client_server_map, + process_id, + secret_key, + client_server_map, parameters: HashMap::new(), stats: get_reporter(), admin: false, last_address_id: None, last_server_id: None, - target_pool_name: String::from("undefined"), - target_user_name: String::from("undefined"), - shutdown_event_receiver: shutdown_event_receiver, + pool_name: String::from("undefined"), + username: String::from("undefined"), + shutdown, connected_to_server: false, }); } @@ -486,7 +548,7 @@ where process_id.clone(), secret_key.clone(), address.clone(), - port.clone(), + *port, ), // The client doesn't know / got the wrong server, @@ -498,7 +560,7 @@ where // Opens a new separate connection to the server, sends the backend_id // and secret_key and then closes it for security reasons. No other interactions // take place. - return Ok(Server::cancel(&address, &port, process_id, secret_key).await?); + return Ok(Server::cancel(&address, port, process_id, secret_key).await?); } // The query router determines where the query is going to go, @@ -521,9 +583,19 @@ where // SET SHARDING KEY TO 'bigint'; let mut message = tokio::select! { - _ = self.shutdown_event_receiver.recv() => { - error_response_terminal(&mut self.write, &format!("terminating connection due to administrator command")).await?; - return Ok(()) + _ = self.shutdown.recv() => { + if !self.admin { + error_response_terminal( + &mut self.write, + &format!("terminating connection due to administrator command") + ).await?; + return Ok(()) + } + + // Admin clients ignore shutdown. + else { + read_message(&mut self.read).await? + } }, message_result = read_message(&mut self.read) => message_result? }; @@ -544,15 +616,14 @@ where // Get a pool instance referenced by the most up-to-date // pointer. This ensures we always read the latest config // when starting a query. - let pool = match get_pool(self.target_pool_name.clone(), self.target_user_name.clone()) - { + let pool = match get_pool(self.pool_name.clone(), self.username.clone()) { Some(pool) => pool, None => { error_response( &mut self.write, &format!( "No pool configured for database: {:?}, user: {:?}", - self.target_pool_name, self.target_user_name + self.pool_name, self.username ), ) .await?; @@ -649,14 +720,16 @@ where match message[0] as char { 'P' | 'B' | 'E' | 'D' => (), _ => { - error!("Could not get connection from pool: {:?}", err); error_response( &mut self.write, "could not get connection from the pool", ) .await?; } - } + }; + + error!("Could not get connection from pool: {:?}", err); + continue; } }; @@ -739,15 +812,8 @@ where 'Q' => { debug!("Sending query to server"); - self.send_and_receive_loop( - code, - original, - server, - &address, - query_router.shard(), - &pool, - ) - .await?; + self.send_and_receive_loop(code, original, server, &address, &pool) + .await?; if !server.in_transaction() { // Report transaction executed statistics. @@ -814,7 +880,6 @@ where self.buffer.clone(), server, &address, - query_router.shard(), &pool, ) .await?; @@ -836,32 +901,18 @@ where 'd' => { // Forward the data to the server, // don't buffer it since it can be rather large. - self.send_server_message( - server, - original, - &address, - query_router.shard(), - &pool, - ) - .await?; + self.send_server_message(server, original, &address, &pool) + .await?; } // CopyDone or CopyFail // Copy is done, successfully or not. 'c' | 'f' => { - self.send_server_message( - server, - original, - &address, - query_router.shard(), - &pool, - ) - .await?; - - let response = self - .receive_server_message(server, &address, query_router.shard(), &pool) + self.send_server_message(server, original, &address, &pool) .await?; + let response = self.receive_server_message(server, &address, &pool).await?; + match write_all_half(&mut self.write, response).await { Ok(_) => (), Err(err) => { @@ -910,20 +961,17 @@ where message: BytesMut, server: &mut Server, address: &Address, - shard: usize, pool: &ConnectionPool, ) -> Result<(), Error> { debug!("Sending {} to server", code); - self.send_server_message(server, message, &address, shard, &pool) + self.send_server_message(server, message, &address, &pool) .await?; // Read all data the server has to offer, which can be multiple messages // buffered in 8196 bytes chunks. loop { - let response = self - .receive_server_message(server, &address, shard, &pool) - .await?; + let response = self.receive_server_message(server, &address, &pool).await?; match write_all_half(&mut self.write, response).await { Ok(_) => (), @@ -949,13 +997,12 @@ where server: &mut Server, message: BytesMut, address: &Address, - shard: usize, pool: &ConnectionPool, ) -> Result<(), Error> { match server.send(message).await { Ok(_) => Ok(()), Err(err) => { - pool.ban(address, shard, self.process_id); + pool.ban(address, self.process_id); Err(err) } } @@ -965,7 +1012,6 @@ where &mut self, server: &mut Server, address: &Address, - shard: usize, pool: &ConnectionPool, ) -> Result { if pool.settings.user.statement_timeout > 0 { @@ -978,7 +1024,7 @@ where Ok(result) => match result { Ok(message) => Ok(message), Err(err) => { - pool.ban(address, shard, self.process_id); + pool.ban(address, self.process_id); error_response_terminal( &mut self.write, &format!("error receiving data from server: {:?}", err), @@ -993,7 +1039,7 @@ where address, pool.settings.user.username ); server.mark_bad(); - pool.ban(address, shard, self.process_id); + pool.ban(address, self.process_id); error_response_terminal(&mut self.write, "pool statement timeout").await?; Err(Error::StatementTimeout) } @@ -1002,7 +1048,7 @@ where match server.recv().await { Ok(message) => Ok(message), Err(err) => { - pool.ban(address, shard, self.process_id); + pool.ban(address, self.process_id); error_response_terminal( &mut self.write, &format!("error receiving data from server: {:?}", err), diff --git a/src/config.rs b/src/config.rs index b75169373..5c1226117 100644 --- a/src/config.rs +++ b/src/config.rs @@ -57,15 +57,34 @@ impl PartialEq for Option { /// Address identifying a PostgreSQL server uniquely. #[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)] pub struct Address { + /// Unique ID per addressable Postgres server. pub id: usize, + + /// Server host. pub host: String, - pub port: String, + + /// Server port. + pub port: u16, + + /// Shard number of this Postgres server. pub shard: usize, + + /// The name of the Postgres database. pub database: String, + + /// Server role: replica, primary. pub role: Role, + + /// If it's a replica, number it for reference and failover. pub replica_number: usize, + + /// Position of the server in the pool for failover. pub address_index: usize, + + /// The name of the user configured to use this pool. pub username: String, + + /// The name of this pool (i.e. database name visible to the client). pub pool_name: String, } @@ -74,7 +93,7 @@ impl Default for Address { Address { id: 0, host: String::from("127.0.0.1"), - port: String::from("5432"), + port: 5432, shard: 0, address_index: 0, replica_number: 0, @@ -341,9 +360,9 @@ impl Config { for (pool_name, pool_config) in &self.pools { // TODO: Make this output prettier (maybe a table?) - info!("--- Settings for pool {} ---", pool_name); info!( - "Pool size from all users: {}", + "[pool: {}] Maximum user connections: {}", + pool_name, pool_config .users .iter() @@ -351,20 +370,39 @@ impl Config { .sum::() .to_string() ); - info!("Pool mode: {}", pool_config.pool_mode); - info!("Sharding function: {}", pool_config.sharding_function); - info!("Primary reads: {}", pool_config.primary_reads_enabled); - info!("Query router: {}", pool_config.query_parser_enabled); - - // TODO: Make this prettier. - info!("Number of shards: {}", pool_config.shards.len()); - info!("Number of users: {}", pool_config.users.len()); + info!("[pool: {}] Pool mode: {}", pool_name, pool_config.pool_mode); + info!( + "[pool: {}] Sharding function: {}", + pool_name, pool_config.sharding_function + ); + info!( + "[pool: {}] Primary reads: {}", + pool_name, pool_config.primary_reads_enabled + ); + info!( + "[pool: {}] Query router: {}", + pool_name, pool_config.query_parser_enabled + ); + info!( + "[pool: {}] Number of shards: {}", + pool_name, + pool_config.shards.len() + ); + info!( + "[pool: {}] Number of users: {}", + pool_name, + pool_config.users.len() + ); for user in &pool_config.users { info!( - "{} pool size: {}, statement timeout: {}", - user.1.username, user.1.pool_size, user.1.statement_timeout + "[pool: {}][user: {}] Pool size: {}", + pool_name, user.1.username, user.1.pool_size, ); + info!( + "[pool: {}][user: {}] Statement timeout: {}", + pool_name, user.1.username, user.1.statement_timeout + ) } } } @@ -462,6 +500,18 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; + match pool.pool_mode.as_ref() { + "transaction" => (), + "session" => (), + other => { + error!( + "pool_mode can be 'session' or 'transaction', got: '{}'", + other + ); + return Err(Error::BadConfig); + } + }; + for shard in &pool.shards { // We use addresses as unique identifiers, // let's make sure they are unique in the config as well. diff --git a/src/errors.rs b/src/errors.rs index 06371fd19..50301f366 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -12,4 +12,5 @@ pub enum Error { ClientError, TlsError, StatementTimeout, + ShuttingDown, } diff --git a/src/main.rs b/src/main.rs index 0b2e1d59f..9aad61a13 100644 --- a/src/main.rs +++ b/src/main.rs @@ -66,6 +66,7 @@ mod stats; mod tls; use crate::config::{get_config, reload_config, VERSION}; +use crate::errors::Error; use crate::pool::{ClientServerMap, ConnectionPool}; use crate::prometheus::start_metric_server; use crate::stats::{Collector, Reporter, REPORTER}; @@ -133,8 +134,8 @@ async fn main() { let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); // Statistics reporting. - let (tx, rx) = mpsc::channel(100_000); - REPORTER.store(Arc::new(Reporter::new(tx.clone()))); + let (stats_tx, stats_rx) = mpsc::channel(100_000); + REPORTER.store(Arc::new(Reporter::new(stats_tx.clone()))); // Connection pool that allows to query all shards and replicas. match ConnectionPool::from_config(client_server_map.clone()).await { @@ -145,159 +146,148 @@ async fn main() { } }; - // Statistics collector task. - let collector_tx = tx.clone(); - - // Save these for reloading - let reload_client_server_map = client_server_map.clone(); - let autoreload_client_server_map = client_server_map.clone(); - tokio::task::spawn(async move { - let mut stats_collector = Collector::new(rx, collector_tx); + let mut stats_collector = Collector::new(stats_rx, stats_tx.clone()); stats_collector.collect().await; }); + info!("Config autoreloader: {}", config.general.autoreload); + + let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); + let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); + let mut sighup_signal = unix_signal(SignalKind::hangup()).unwrap(); + let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); + let (shutdown_tx, _) = broadcast::channel::<()>(1); + let (drain_tx, mut drain_rx) = mpsc::channel::(2048); + let (exit_tx, mut exit_rx) = mpsc::channel::<()>(1); + info!("Waiting for clients"); - let (shutdown_event_tx, mut shutdown_event_rx) = broadcast::channel::<()>(1); + let mut admin_only = false; + let mut total_clients = 0; - let shutdown_event_tx_clone = shutdown_event_tx.clone(); + loop { + tokio::select! { + // Reload config: + // kill -SIGHUP $(pgrep pgcat) + _ = sighup_signal.recv() => { + info!("Reloading config"); - // Client connection loop. - tokio::task::spawn(async move { - // Creates event subscriber for shutdown event, this is dropped when shutdown event is broadcast - let mut listener_shutdown_event_rx = shutdown_event_tx_clone.subscribe(); - loop { - let client_server_map = client_server_map.clone(); - - // Listen for shutdown event and client connection at the same time - let (socket, addr) = tokio::select! { - _ = listener_shutdown_event_rx.recv() => { - // Exits client connection loop which drops listener, listener_shutdown_event_rx and shutdown_event_tx_clone - break; - } + match reload_config(client_server_map.clone()).await { + Ok(_) => (), + Err(_) => (), + }; - listener_response = listener.accept() => { - match listener_response { - Ok((socket, addr)) => (socket, addr), - Err(err) => { - error!("{:?}", err); - continue; + get_config().show(); + }, + + _ = autoreload_interval.tick() => { + if config.general.autoreload { + info!("Automatically reloading config"); + + match reload_config(client_server_map.clone()).await { + Ok(changed) => { + if changed { + get_config().show() + } } - } + Err(_) => (), + }; } - }; - - // Used to signal shutdown - let client_shutdown_handler_rx = shutdown_event_tx_clone.subscribe(); - - // Used to signal that the task has completed - let dummy_tx = shutdown_event_tx_clone.clone(); - - // Handle client. - tokio::task::spawn(async move { - let start = chrono::offset::Utc::now().naive_utc(); - - match client::client_entrypoint( - socket, - client_server_map, - client_shutdown_handler_rx, - ) - .await - { - Ok(_) => { - let duration = chrono::offset::Utc::now().naive_utc() - start; - - info!( - "Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); - } + }, - Err(err) => { - debug!("Client disconnected with error {:?}", err); - } - }; - // Drop this transmitter so receiver knows that the task is completed - drop(dummy_tx); - }); - } - }); + // Initiate graceful shutdown sequence on sig int + _ = interrupt_signal.recv() => { + info!("Got SIGINT, waiting for client connection drain now"); + admin_only = true; - // Reload config: - // kill -SIGHUP $(pgrep pgcat) - tokio::task::spawn(async move { - let mut stream = unix_signal(SignalKind::hangup()).unwrap(); + // Broadcast that client tasks need to finish + let _ = shutdown_tx.send(()); + let exit_tx = exit_tx.clone(); + let _ = drain_tx.send(0).await; - loop { - stream.recv().await; + tokio::task::spawn(async move { + let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(config.general.shutdown_timeout)); - info!("Reloading config"); + // First tick fires immediately. + interval.tick().await; - match reload_config(reload_client_server_map.clone()).await { - Ok(_) => (), - Err(_) => continue, - }; + // Second one in the interval time. + interval.tick().await; - get_config().show(); - } - }); + // We're done waiting. + error!("Timed out waiting for clients"); - if config.general.autoreload { - let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); + let _ = exit_tx.send(()).await; + }); + }, - tokio::task::spawn(async move { - info!("Config autoreloader started"); - - loop { - interval.tick().await; - match reload_config(autoreload_client_server_map.clone()).await { - Ok(changed) => { - if changed { - get_config().show() - } + _ = term_signal.recv() => break, + + new_client = listener.accept() => { + let (socket, addr) = match new_client { + Ok((socket, addr)) => (socket, addr), + Err(err) => { + error!("{:?}", err); + continue; } - Err(_) => (), }; + + let shutdown_rx = shutdown_tx.subscribe(); + let drain_tx = drain_tx.clone(); + let client_server_map = client_server_map.clone(); + + tokio::task::spawn(async move { + let start = chrono::offset::Utc::now().naive_utc(); + + match client::client_entrypoint( + socket, + client_server_map, + shutdown_rx, + drain_tx, + admin_only, + ) + .await + { + Ok(()) => { + + let duration = chrono::offset::Utc::now().naive_utc() - start; + + info!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + } + + Err(err) => { + match err { + // Don't count the clients we rejected. + Error::ShuttingDown => (), + _ => { + // drain_tx.send(-1).await.unwrap(); + } + } + + debug!("Client disconnected with error {:?}", err); + } + }; + }); } - }); - } - let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); - let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); + _ = exit_rx.recv() => { + break; + } - tokio::select! { - // Initiate graceful shutdown sequence on sig int - _ = interrupt_signal.recv() => { - info!("Got SIGINT, waiting for client connection drain now"); - - // Broadcast that client tasks need to finish - shutdown_event_tx.send(()).unwrap(); - // Closes transmitter - drop(shutdown_event_tx); - - // This is in a loop because the first event that the receiver receives will be the shutdown event - // This is not what we are waiting for instead, we want the receiver to send an error once all senders are closed which is reached after the shutdown event is received - loop { - match tokio::time::timeout( - tokio::time::Duration::from_millis(config.general.shutdown_timeout), - shutdown_event_rx.recv(), - ) - .await - { - Ok(res) => match res { - Ok(_) => {} - Err(_) => break, - }, - Err(_) => { - info!("Timed out while waiting for clients to shutdown"); - break; - } + client_ping = drain_rx.recv() => { + let client_ping = client_ping.unwrap(); + total_clients += client_ping; + + if total_clients == 0 && admin_only { + let _ = exit_tx.send(()).await; } } - }, - _ = term_signal.recv() => (), + } } info!("Shutting down..."); diff --git a/src/pool.rs b/src/pool.rs index ac5bc9110..99cccaf10 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -12,40 +12,74 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; -use crate::config::{get_config, Address, Role, Shard, User}; +use crate::config::{get_config, Address, Role, User}; use crate::errors::Error; use crate::server::Server; +use crate::sharding::ShardingFunction; use crate::stats::{get_reporter, Reporter}; pub type BanList = Arc>>>; -pub type ClientServerMap = Arc>>; +pub type ClientServerMap = Arc>>; pub type PoolMap = HashMap<(String, String), ConnectionPool>; /// The connection pool, globally available. /// This is atomic and safe and read-optimized. /// The pool is recreated dynamically when the config is reloaded. pub static POOLS: Lazy> = Lazy::new(|| ArcSwap::from_pointee(HashMap::default())); +/// Pool mode: +/// - transaction: server serves one transaction, +/// - session: server is attached to the client. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum PoolMode { + Session, + Transaction, +} + +impl std::fmt::Display for PoolMode { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match *self { + PoolMode::Session => write!(f, "session"), + PoolMode::Transaction => write!(f, "transaction"), + } + } +} + +/// Pool settings. #[derive(Clone, Debug)] pub struct PoolSettings { - pub pool_mode: String, - pub shards: HashMap, + /// Transaction or Session. + pub pool_mode: PoolMode, + + // Number of shards. + pub shards: usize, + + // Connecting user. pub user: User, - pub default_role: String, + + // Default server role to connect to. + pub default_role: Option, + + // Enable/disable query parser. pub query_parser_enabled: bool, + + // Read from the primary as well or not. pub primary_reads_enabled: bool, - pub sharding_function: String, + + // Sharding function. + pub sharding_function: ShardingFunction, } + impl Default for PoolSettings { fn default() -> PoolSettings { PoolSettings { - pool_mode: String::from("transaction"), - shards: HashMap::from([(String::from("1"), Shard::default())]), + pool_mode: PoolMode::Transaction, + shards: 1, user: User::default(), - default_role: String::from("any"), + default_role: None, query_parser_enabled: false, primary_reads_enabled: true, - sharding_function: "pg_bigint_hash".to_string(), + sharding_function: ShardingFunction::PgBigintHash, } } } @@ -73,6 +107,7 @@ pub struct ConnectionPool { /// on pool creation and save the K messages here. server_info: BytesMut, + /// Pool configuration. pub settings: PoolSettings, } @@ -80,11 +115,13 @@ impl ConnectionPool { /// Construct the connection pool from the configuration. pub async fn from_config(client_server_map: ClientServerMap) -> Result<(), Error> { let config = get_config(); - let mut new_pools = PoolMap::default(); + let mut new_pools = HashMap::new(); let mut address_id = 0; + for (pool_name, pool_config) in &config.pools { - for (_user_index, user_info) in &pool_config.users { + // There is one pool per database/user pair. + for (_, user) in &pool_config.users { let mut shards = Vec::new(); let mut addresses = Vec::new(); let mut banlist = Vec::new(); @@ -98,8 +135,8 @@ impl ConnectionPool { // Sort by shard number to ensure consistency. shard_ids.sort_by_key(|k| k.parse::().unwrap()); - for shard_idx in shard_ids { - let shard = &pool_config.shards[&shard_idx]; + for shard_idx in &shard_ids { + let shard = &pool_config.shards[shard_idx]; let mut pools = Vec::new(); let mut servers = Vec::new(); let mut address_index = 0; @@ -119,12 +156,12 @@ impl ConnectionPool { id: address_id, database: shard.database.clone(), host: server.0.clone(), - port: server.1.to_string(), + port: server.1 as u16, role: role, address_index, replica_number, shard: shard_idx.parse::().unwrap(), - username: user_info.username.clone(), + username: user.username.clone(), pool_name: pool_name.clone(), }; @@ -137,14 +174,14 @@ impl ConnectionPool { let manager = ServerPool::new( address.clone(), - user_info.clone(), + user.clone(), &shard.database, client_server_map.clone(), get_reporter(), ); let pool = Pool::builder() - .max_size(user_info.pool_size) + .max_size(user.pool_size) .connection_timeout(std::time::Duration::from_millis( config.general.connect_timeout, )) @@ -171,13 +208,27 @@ impl ConnectionPool { stats: get_reporter(), server_info: BytesMut::new(), settings: PoolSettings { - pool_mode: pool_config.pool_mode.clone(), - shards: pool_config.shards.clone(), - user: user_info.clone(), - default_role: pool_config.default_role.clone(), + pool_mode: match pool_config.pool_mode.as_str() { + "transaction" => PoolMode::Transaction, + "session" => PoolMode::Session, + _ => unreachable!(), + }, + // shards: pool_config.shards.clone(), + shards: shard_ids.len(), + user: user.clone(), + default_role: match pool_config.default_role.as_str() { + "any" => None, + "replica" => Some(Role::Replica), + "primary" => Some(Role::Primary), + _ => unreachable!(), + }, query_parser_enabled: pool_config.query_parser_enabled.clone(), primary_reads_enabled: pool_config.primary_reads_enabled, - sharding_function: pool_config.sharding_function.clone(), + sharding_function: match pool_config.sharding_function.as_str() { + "pg_bigint_hash" => ShardingFunction::PgBigintHash, + "sha1" => ShardingFunction::Sha1, + _ => unreachable!(), + }, }, }; @@ -190,7 +241,9 @@ impl ConnectionPool { return Err(err); } }; - new_pools.insert((pool_name.clone(), user_info.username.clone()), pool); + + // There is one pool per database/user pair. + new_pools.insert((pool_name.clone(), user.username.clone()), pool); } } @@ -207,8 +260,8 @@ impl ConnectionPool { async fn validate(&mut self) -> Result<(), Error> { let mut server_infos = Vec::new(); for shard in 0..self.shards() { - for index in 0..self.servers(shard) { - let connection = match self.databases[shard][index].get().await { + for server in 0..self.servers(shard) { + let connection = match self.databases[shard][server].get().await { Ok(conn) => conn, Err(err) => { error!("Shard {} down or misconfigured: {:?}", shard, err); @@ -229,6 +282,7 @@ impl ConnectionPool { ); } } + server_infos.push(server_info); } } @@ -239,6 +293,8 @@ impl ConnectionPool { return Err(Error::AllServersDown); } + // We're assuming all servers are identical. + // TODO: not true. self.server_info = server_infos[0].clone(); Ok(()) @@ -252,9 +308,8 @@ impl ConnectionPool { process_id: i32, // client id ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { let now = Instant::now(); - let mut candidates: Vec
= self.addresses[shard] - .clone() - .into_iter() + let mut candidates: Vec<&Address> = self.addresses[shard] + .iter() .filter(|address| address.role == role) .collect(); @@ -271,7 +326,8 @@ impl ConnectionPool { None => break, }; - if self.is_banned(&address, address.shard, role) { + if self.is_banned(&address, role) { + debug!("Address {:?} is banned", address); continue; } @@ -286,8 +342,7 @@ impl ConnectionPool { Ok(conn) => conn, Err(err) => { error!("Banning instance {:?}, error: {:?}", address, err); - self.ban(&address, address.shard, process_id); - self.stats.client_disconnecting(process_id, address.id); + self.ban(&address, process_id); self.stats .checkout_time(now.elapsed().as_micros(), process_id, address.id); continue; @@ -301,6 +356,9 @@ impl ConnectionPool { let require_healthcheck = server.last_activity().elapsed().unwrap().as_millis() > healthcheck_delay; + // Do not issue a health check unless it's been a little while + // since we last checked the server is ok. + // Health checks are pretty expensive. if !require_healthcheck { self.stats .checkout_time(now.elapsed().as_micros(), process_id, address.id); @@ -314,7 +372,7 @@ impl ConnectionPool { match tokio::time::timeout( tokio::time::Duration::from_millis(healthcheck_timeout), - server.query(";"), + server.query(";"), // Cheap query (query parser not used in PG) ) .await { @@ -337,7 +395,7 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(&address, address.shard, process_id); + self.ban(&address, process_id); continue; } }, @@ -351,44 +409,44 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(&address, address.shard, process_id); + self.ban(&address, process_id); continue; } } } - return Err(Error::AllServersDown); + + Err(Error::AllServersDown) } /// Ban an address (i.e. replica). It no longer will serve /// traffic for any new transactions. Existing transactions on that replica /// will finish successfully or error out to the clients. - pub fn ban(&self, address: &Address, shard: usize, process_id: i32) { + pub fn ban(&self, address: &Address, process_id: i32) { self.stats.client_disconnecting(process_id, address.id); - self.stats - .checkout_time(Instant::now().elapsed().as_micros(), process_id, address.id); error!("Banning {:?}", address); + let now = chrono::offset::Utc::now().naive_utc(); let mut guard = self.banlist.write(); - guard[shard].insert(address.clone(), now); + guard[address.shard].insert(address.clone(), now); } /// Clear the replica to receive traffic again. Takes effect immediately /// for all new transactions. - pub fn _unban(&self, address: &Address, shard: usize) { + pub fn _unban(&self, address: &Address) { let mut guard = self.banlist.write(); - guard[shard].remove(address); + guard[address.shard].remove(address); } /// Check if a replica can serve traffic. If all replicas are banned, /// we unban all of them. Better to try then not to. - pub fn is_banned(&self, address: &Address, shard: usize, role: Option) -> bool { + pub fn is_banned(&self, address: &Address, role: Option) -> bool { let replicas_available = match role { - Some(Role::Replica) => self.addresses[shard] + Some(Role::Replica) => self.addresses[address.shard] .iter() .filter(|addr| addr.role == Role::Replica) .count(), - None => self.addresses[shard].len(), + None => self.addresses[address.shard].len(), Some(Role::Primary) => return false, // Primary cannot be banned. }; @@ -397,17 +455,17 @@ impl ConnectionPool { let guard = self.banlist.read(); // Everything is banned = nothing is banned. - if guard[shard].len() == replicas_available { + if guard[address.shard].len() == replicas_available { drop(guard); let mut guard = self.banlist.write(); - guard[shard].clear(); + guard[address.shard].clear(); drop(guard); warn!("Unbanning all replicas."); return false; } // I expect this to miss 99.9999% of the time. - match guard[shard].get(address) { + match guard[address.shard].get(address) { Some(timestamp) => { let now = chrono::offset::Utc::now().naive_utc(); let config = get_config(); @@ -417,7 +475,7 @@ impl ConnectionPool { drop(guard); warn!("Unbanning {:?}", address); let mut guard = self.banlist.write(); - guard[shard].remove(address); + guard[address.shard].remove(address); false } else { debug!("{:?} is banned", address); @@ -554,6 +612,7 @@ pub fn get_pool(db: String, user: String) -> Option { } } +/// How many total servers we have in the config. pub fn get_number_of_addresses() -> usize { get_all_pools() .iter() @@ -561,6 +620,7 @@ pub fn get_number_of_addresses() -> usize { .sum() } +/// Get a pointer to all configured pools. pub fn get_all_pools() -> HashMap<(String, String), ConnectionPool> { return (*(*POOLS.load())).clone(); } diff --git a/src/query_router.rs b/src/query_router.rs index 6b377684f..f9d5f0b37 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -10,7 +10,7 @@ use sqlparser::parser::Parser; use crate::config::Role; use crate::pool::PoolSettings; -use crate::sharding::{Sharder, ShardingFunction}; +use crate::sharding::Sharder; /// Regexes used to parse custom commands. const CUSTOM_SQL_REGEXES: [&str; 7] = [ @@ -55,11 +55,13 @@ pub struct QueryRouter { /// Include the primary into the replica pool for reads. primary_reads_enabled: bool, + /// Pool configuration. pool_settings: PoolSettings, } impl QueryRouter { - /// One-time initialization of regexes. + /// One-time initialization of regexes + /// that parse our custom SQL protocol. pub fn setup() -> bool { let set = match RegexSet::new(&CUSTOM_SQL_REGEXES) { Ok(rgx) => rgx, @@ -74,10 +76,7 @@ impl QueryRouter { .map(|rgx| Regex::new(rgx).unwrap()) .collect(); - // Impossible - if list.len() != set.len() { - return false; - } + assert_eq!(list.len(), set.len()); match CUSTOM_SQL_REGEX_LIST.set(list) { Ok(_) => true, @@ -90,7 +89,8 @@ impl QueryRouter { } } - /// Create a new instance of the query router. Each client gets its own. + /// Create a new instance of the query router. + /// Each client gets its own. pub fn new() -> QueryRouter { QueryRouter { active_shard: None, @@ -101,6 +101,7 @@ impl QueryRouter { } } + /// Pool settings can change because of a config reload. pub fn update_pool_settings(&mut self, pool_settings: PoolSettings) { self.pool_settings = pool_settings; } @@ -136,19 +137,6 @@ impl QueryRouter { return None; } - let sharding_function = match self.pool_settings.sharding_function.as_ref() { - "pg_bigint_hash" => ShardingFunction::PgBigintHash, - "sha1" => ShardingFunction::Sha1, - _ => unreachable!(), - }; - - let default_server_role = match self.pool_settings.default_role.as_ref() { - "any" => None, - "primary" => Some(Role::Primary), - "replica" => Some(Role::Replica), - _ => unreachable!(), - }; - let command = match matches[0] { 0 => Command::SetShardingKey, 1 => Command::SetShard, @@ -200,7 +188,10 @@ impl QueryRouter { match command { Command::SetShardingKey => { - let sharder = Sharder::new(self.pool_settings.shards.len(), sharding_function); + let sharder = Sharder::new( + self.pool_settings.shards, + self.pool_settings.sharding_function, + ); let shard = sharder.shard(value.parse::().unwrap()); self.active_shard = Some(shard); value = shard.to_string(); @@ -208,7 +199,7 @@ impl QueryRouter { Command::SetShard => { self.active_shard = match value.to_ascii_uppercase().as_ref() { - "ANY" => Some(rand::random::() % self.pool_settings.shards.len()), + "ANY" => Some(rand::random::() % self.pool_settings.shards), _ => Some(value.parse::().unwrap()), }; } @@ -236,7 +227,7 @@ impl QueryRouter { } "default" => { - self.active_role = default_server_role; + self.active_role = self.pool_settings.default_role; self.query_parser_enabled = self.query_parser_enabled; self.active_role } @@ -367,10 +358,10 @@ impl QueryRouter { #[cfg(test)] mod test { - use std::collections::HashMap; - use super::*; use crate::messages::simple_query; + use crate::pool::PoolMode; + use crate::sharding::ShardingFunction; use bytes::BufMut; #[test] @@ -633,13 +624,13 @@ mod test { QueryRouter::setup(); let pool_settings = PoolSettings { - pool_mode: "transaction".to_string(), - shards: HashMap::default(), + pool_mode: PoolMode::Transaction, + shards: 0, user: crate::config::User::default(), - default_role: Role::Replica.to_string(), + default_role: Some(Role::Replica), query_parser_enabled: true, primary_reads_enabled: false, - sharding_function: "pg_bigint_hash".to_string(), + sharding_function: ShardingFunction::PgBigintHash, }; let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); @@ -661,9 +652,6 @@ mod test { let q2 = simple_query("SET SERVER ROLE TO 'default'"); assert!(qr.try_execute_command(q2) != None); - assert_eq!( - qr.active_role.unwrap().to_string(), - pool_settings.clone().default_role - ); + assert_eq!(qr.active_role.unwrap(), pool_settings.clone().default_role); } } diff --git a/src/server.rs b/src/server.rs index ddf95ce62..3134a65df 100644 --- a/src/server.rs +++ b/src/server.rs @@ -75,7 +75,7 @@ impl Server { stats: Reporter, ) -> Result { let mut stream = - match TcpStream::connect(&format!("{}:{}", &address.host, &address.port)).await { + match TcpStream::connect(&format!("{}:{}", &address.host, address.port)).await { Ok(stream) => stream, Err(err) => { error!("Could not connect to server: {}", err); @@ -342,7 +342,7 @@ impl Server { /// Uses a separate connection that's not part of the connection pool. pub async fn cancel( host: &str, - port: &str, + port: u16, process_id: i32, secret_key: i32, ) -> Result<(), Error> { @@ -529,7 +529,7 @@ impl Server { self.process_id, self.secret_key, self.address.host.clone(), - self.address.port.clone(), + self.address.port, ), ); } diff --git a/tests/python/tests.py b/tests/python/tests.py index a674cee67..092fc8cc9 100644 --- a/tests/python/tests.py +++ b/tests/python/tests.py @@ -14,6 +14,7 @@ def pgcat_start(): pg_cat_send_signal(signal.SIGTERM) os.system("./target/debug/pgcat .circleci/pgcat.toml &") + time.sleep(2) def pg_cat_send_signal(signal: signal.Signals): @@ -27,11 +28,23 @@ def pg_cat_send_signal(signal: signal.Signals): raise Exception("pgcat not closed after SIGTERM") -def connect_normal_db( - autocommit: bool = False, +def connect_db( + autocommit: bool = True, + admin: bool = False, ) -> Tuple[psycopg2.extensions.connection, psycopg2.extensions.cursor]: + + if admin: + user = "admin_user" + password = "admin_pass" + db = "pgcat" + else: + user = "sharding_user" + password = "sharding_user" + db = "sharded_db" + conn = psycopg2.connect( - f"postgres://sharding_user:sharding_user@{PGCAT_HOST}:{PGCAT_PORT}/sharded_db?application_name=testing_pgcat" + f"postgres://{user}:{password}@{PGCAT_HOST}:{PGCAT_PORT}/{db}?application_name=testing_pgcat", + connect_timeout=2, ) conn.autocommit = autocommit cur = conn.cursor() @@ -45,7 +58,7 @@ def cleanup_conn(conn: psycopg2.extensions.connection, cur: psycopg2.extensions. def test_normal_db_access(): - conn, cur = connect_normal_db() + conn, cur = connect_db(autocommit=False) cur.execute("SELECT 1") res = cur.fetchall() print(res) @@ -53,11 +66,7 @@ def test_normal_db_access(): def test_admin_db_access(): - conn = psycopg2.connect( - f"postgres://admin_user:admin_pass@{PGCAT_HOST}:{PGCAT_PORT}/pgcat" - ) - conn.autocommit = True # BEGIN/COMMIT is not supported by admin db - cur = conn.cursor() + conn, cur = connect_db(admin=True) cur.execute("SHOW POOLS") res = cur.fetchall() @@ -67,15 +76,14 @@ def test_admin_db_access(): def test_shutdown_logic(): - ##### NO ACTIVE QUERIES SIGINT HANDLING ##### + # - - - - - - - - - - - - - - - - - - + # NO ACTIVE QUERIES SIGINT HANDLING + # Start pgcat pgcat_start() - # Wait for server to fully start up - time.sleep(2) - # Create client connection and send query (not in transaction) - conn, cur = connect_normal_db(True) + conn, cur = connect_db() cur.execute("BEGIN;") cur.execute("SELECT 1;") @@ -97,17 +105,14 @@ def test_shutdown_logic(): cleanup_conn(conn, cur) pg_cat_send_signal(signal.SIGTERM) - ##### END ##### + # - - - - - - - - - - - - - - - - - - + # HANDLE TRANSACTION WITH SIGINT - ##### HANDLE TRANSACTION WITH SIGINT ##### # Start pgcat pgcat_start() - # Wait for server to fully start up - time.sleep(2) - # Create client connection and begin transaction - conn, cur = connect_normal_db(True) + conn, cur = connect_db() cur.execute("BEGIN;") cur.execute("SELECT 1;") @@ -126,17 +131,97 @@ def test_shutdown_logic(): cleanup_conn(conn, cur) pg_cat_send_signal(signal.SIGTERM) - ##### END ##### + # - - - - - - - - - - - - - - - - - - + # NO NEW NON-ADMIN CONNECTIONS DURING SHUTDOWN + # Start pgcat + pgcat_start() + + # Create client connection and begin transaction + transaction_conn, transaction_cur = connect_db() + + transaction_cur.execute("BEGIN;") + transaction_cur.execute("SELECT 1;") + + # Send sigint to pgcat while still in transaction + pg_cat_send_signal(signal.SIGINT) + time.sleep(1) - ##### HANDLE SHUTDOWN TIMEOUT WITH SIGINT ##### + start = time.perf_counter() + try: + conn, cur = connect_db() + cur.execute("SELECT 1;") + cleanup_conn(conn, cur) + except psycopg2.OperationalError as e: + time_taken = time.perf_counter() - start + if time_taken > 0.1: + raise Exception( + "Failed to reject connection within 0.1 seconds, got", time_taken, "seconds") + pass + else: + raise Exception("Able connect to database during shutdown") + + cleanup_conn(transaction_conn, transaction_cur) + pg_cat_send_signal(signal.SIGTERM) + + # - - - - - - - - - - - - - - - - - - + # ALLOW NEW ADMIN CONNECTIONS DURING SHUTDOWN # Start pgcat pgcat_start() - # Wait for server to fully start up - time.sleep(3) + # Create client connection and begin transaction + transaction_conn, transaction_cur = connect_db() + + transaction_cur.execute("BEGIN;") + transaction_cur.execute("SELECT 1;") + + # Send sigint to pgcat while still in transaction + pg_cat_send_signal(signal.SIGINT) + time.sleep(1) + + try: + conn, cur = connect_db(admin=True) + cur.execute("SHOW DATABASES;") + cleanup_conn(conn, cur) + except psycopg2.OperationalError as e: + raise Exception(e) + + cleanup_conn(transaction_conn, transaction_cur) + pg_cat_send_signal(signal.SIGTERM) + + # - - - - - - - - - - - - - - - - - - + # ADMIN CONNECTIONS CONTINUING TO WORK AFTER SHUTDOWN + # Start pgcat + pgcat_start() + + # Create client connection and begin transaction + transaction_conn, transaction_cur = connect_db() + transaction_cur.execute("BEGIN;") + transaction_cur.execute("SELECT 1;") + + admin_conn, admin_cur = connect_db(admin=True) + admin_cur.execute("SHOW DATABASES;") + + # Send sigint to pgcat while still in transaction + pg_cat_send_signal(signal.SIGINT) + time.sleep(1) + + try: + admin_cur.execute("SHOW DATABASES;") + except psycopg2.OperationalError as e: + raise Exception("Could not execute admin command:", e) + + cleanup_conn(transaction_conn, transaction_cur) + cleanup_conn(admin_conn, admin_cur) + pg_cat_send_signal(signal.SIGTERM) + + # - - - - - - - - - - - - - - - - - - + # HANDLE SHUTDOWN TIMEOUT WITH SIGINT + + # Start pgcat + pgcat_start() # Create client connection and begin transaction, which should prevent server shutdown unless shutdown timeout is reached - conn, cur = connect_normal_db(True) + conn, cur = connect_db() cur.execute("BEGIN;") cur.execute("SELECT 1;") @@ -159,7 +244,7 @@ def test_shutdown_logic(): cleanup_conn(conn, cur) pg_cat_send_signal(signal.SIGTERM) - ##### END ##### + # - - - - - - - - - - - - - - - - - - test_normal_db_access() From 10c6038d3b768bbb571ba5b646d8633b2e744f6f Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 25 Aug 2022 20:56:18 -0500 Subject: [PATCH 193/551] Exit with failure codes if configs are bad (#146) * Exit with failure codes if configs are bad * fmt --- .circleci/run_tests.sh | 3 +++ Cargo.lock | 7 +++++++ Cargo.toml | 1 + src/main.rs | 11 ++++++----- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 4202eb5ed..48303d707 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -3,6 +3,9 @@ set -e set -o xtrace +# non-zero exit code if we provide bad configs +(! ./target/debug/pgcat "fake_configs" 2>/dev/null) + # Start PgCat with a particular log level # for inspection. function start_pgcat() { diff --git a/Cargo.lock b/Cargo.lock index 2e20f7095..af4826b87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -159,6 +159,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "exitcode" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" + [[package]] name = "fnv" version = "1.0.7" @@ -515,6 +521,7 @@ dependencies = [ "bytes", "chrono", "env_logger", + "exitcode", "hmac", "hyper", "log", diff --git a/Cargo.toml b/Cargo.toml index 373702440..226a20d6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,3 +33,4 @@ tokio-rustls = "0.23" rustls-pemfile = "1" hyper = { version = "0.14", features = ["full"] } phf = { version = "0.10", features = ["macros"] } +exitcode = "1.1.2" diff --git a/src/main.rs b/src/main.rs index 9aad61a13..72dc70738 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,7 @@ extern crate async_trait; extern crate bb8; extern crate bytes; extern crate env_logger; +extern crate exitcode; extern crate log; extern crate md5; extern crate num_cpus; @@ -78,7 +79,7 @@ async fn main() { if !query_router::QueryRouter::setup() { error!("Could not setup query router"); - return; + std::process::exit(exitcode::CONFIG); } let args = std::env::args().collect::>(); @@ -93,7 +94,7 @@ async fn main() { Ok(_) => (), Err(err) => { error!("Config parse error: {:?}", err); - return; + std::process::exit(exitcode::CONFIG); } }; @@ -108,7 +109,7 @@ async fn main() { Ok(addr) => addr, Err(err) => { error!("Invalid http address: {}", err); - return; + std::process::exit(exitcode::CONFIG); } }; tokio::task::spawn(async move { @@ -122,7 +123,7 @@ async fn main() { Ok(sock) => sock, Err(err) => { error!("Listener socket error: {:?}", err); - return; + std::process::exit(exitcode::CONFIG); } }; @@ -142,7 +143,7 @@ async fn main() { Ok(_) => (), Err(err) => { error!("Pool error: {:?}", err); - return; + std::process::exit(exitcode::CONFIG); } }; From 0dcff758954625fde43dc1a04921d8ab923cc766 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Mon, 29 Aug 2022 03:08:44 -0400 Subject: [PATCH 194/551] Move autoreloader to own tokio task (#148) --- src/main.rs | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/main.rs b/src/main.rs index 72dc70738..a0c1d7cf4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -154,10 +154,29 @@ async fn main() { info!("Config autoreloader: {}", config.general.autoreload); + let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); + let autoreload_client_server_map = client_server_map.clone(); + tokio::task::spawn(async move { + loop { + autoreload_interval.tick().await; + if config.general.autoreload { + info!("Automatically reloading config"); + + match reload_config(autoreload_client_server_map.clone()).await { + Ok(changed) => { + if changed { + get_config().show() + } + } + Err(_) => (), + }; + } + } + }); + let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); let mut sighup_signal = unix_signal(SignalKind::hangup()).unwrap(); - let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); let (shutdown_tx, _) = broadcast::channel::<()>(1); let (drain_tx, mut drain_rx) = mpsc::channel::(2048); let (exit_tx, mut exit_rx) = mpsc::channel::<()>(1); @@ -182,21 +201,6 @@ async fn main() { get_config().show(); }, - _ = autoreload_interval.tick() => { - if config.general.autoreload { - info!("Automatically reloading config"); - - match reload_config(client_server_map.clone()).await { - Ok(changed) => { - if changed { - get_config().show() - } - } - Err(_) => (), - }; - } - }, - // Initiate graceful shutdown sequence on sig int _ = interrupt_signal.recv() => { info!("Got SIGINT, waiting for client connection drain now"); From 5094e32f810ef1f636d2942b430d7954e539339c Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 30 Aug 2022 11:14:53 -0500 Subject: [PATCH 195/551] Ruby integration tests (#147) * Ruby integration tests * forgot a file * refactor * refactoring * more refactoring * remove config helper * try multiple databases * fix * more databases * Use pg stats * ports * speed * Fix tests * preload library * comment --- .circleci/config.yml | 26 ++++- .circleci/run_tests.sh | 14 ++- tests/ruby/Gemfile | 6 +- tests/ruby/Gemfile.lock | 22 +++- tests/ruby/helpers/pg_instance.rb | 82 +++++++++++++ tests/ruby/helpers/pgcat_helper.rb | 100 ++++++++++++++++ tests/ruby/helpers/pgcat_process.rb | 116 +++++++++++++++++++ tests/ruby/load_balancing_spec.rb | 61 ++++++++++ tests/ruby/misc_spec.rb | 109 +++++++++++++++++ tests/ruby/routing_spec.rb | 81 +++++++++++++ tests/ruby/spec_helper.rb | 21 ++++ tests/ruby/tests.rb | 154 +------------------------ tests/sharding/query_routing_setup.sql | 12 ++ 13 files changed, 639 insertions(+), 165 deletions(-) create mode 100644 tests/ruby/helpers/pg_instance.rb create mode 100644 tests/ruby/helpers/pgcat_helper.rb create mode 100644 tests/ruby/helpers/pgcat_process.rb create mode 100644 tests/ruby/load_balancing_spec.rb create mode 100644 tests/ruby/misc_spec.rb create mode 100644 tests/ruby/routing_spec.rb create mode 100644 tests/ruby/spec_helper.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d3449a1a..98aaea714 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,14 +15,34 @@ jobs: RUSTFLAGS: "-C instrument-coverage" LLVM_PROFILE_FILE: "pgcat-%m.profraw" - image: postgres:14 - # auth: - # username: mydockerhub-user - # password: $DOCKERHUB_PASSWORD + command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements"] environment: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + - image: postgres:14 + command: ["postgres", "-p", "7432", "-c", "shared_preload_libraries=pg_stat_statements"] + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + - image: postgres:14 + command: ["postgres", "-p", "8432", "-c", "shared_preload_libraries=pg_stat_statements"] + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + - image: postgres:14 + command: ["postgres", "-p", "9432", "-c", "shared_preload_libraries=pg_stat_statements"] + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 48303d707..1585ebd8e 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -16,6 +16,9 @@ function start_pgcat() { # Setup the database with shards and user PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/query_routing_setup.sql +PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 7432 -U postgres -f tests/sharding/query_routing_setup.sql +PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 8432 -U postgres -f tests/sharding/query_routing_setup.sql +PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 9432 -U postgres -f tests/sharding/query_routing_setup.sql PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard0 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i @@ -26,7 +29,7 @@ wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/downlo sudo dpkg -i toxiproxy-2.4.0.deb # Start Toxiproxy -toxiproxy-server & +LOG_LEVEL=error toxiproxy-server & sleep 1 # Create a database at port 5433, forward it to Postgres @@ -87,7 +90,8 @@ kill -SIGHUP $(pgrep pgcat) # Reload config again cd tests/ruby sudo gem install bundler bundle install -ruby tests.rb +bundle exec ruby tests.rb +bundle exec rspec *_spec.rb cd ../.. # @@ -105,9 +109,9 @@ psql -U admin_user -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/n psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' > /dev/null psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW CONFIG' > /dev/null psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' > /dev/null -psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW LISTS' > /dev/null -psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW POOLS' > /dev/null -psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW VERSION' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgcat -c 'SHOW LISTS' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgcat -c 'SHOW POOLS' > /dev/null +psql -U admin_user -h 127.0.0.1 -p 6432 -d pgcat -c 'SHOW VERSION' > /dev/null psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c "SET client_encoding TO 'utf8'" > /dev/null # will ignore (! psql -U admin_user -e -h 127.0.0.1 -p 6432 -d random_db -c 'SHOW STATS' > /dev/null) export PGPASSWORD=sharding_user diff --git a/tests/ruby/Gemfile b/tests/ruby/Gemfile index 7b019183e..ec7cd3a75 100644 --- a/tests/ruby/Gemfile +++ b/tests/ruby/Gemfile @@ -1,6 +1,8 @@ source "https://rubygems.org" gem "pg" -gem "activerecord" +gem "toml" +gem "rspec" gem "rubocop" -gem "toml", "~> 0.3.0" +gem "toxiproxy" +gem "activerecord" diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock index 3fd03471f..65d8bce7a 100644 --- a/tests/ruby/Gemfile.lock +++ b/tests/ruby/Gemfile.lock @@ -13,6 +13,7 @@ GEM tzinfo (~> 2.0) ast (2.4.2) concurrent-ruby (1.1.10) + diff-lcs (1.5.0) i18n (1.11.0) concurrent-ruby (~> 1.0) minitest (5.16.2) @@ -24,6 +25,19 @@ GEM rainbow (3.1.1) regexp_parser (2.3.1) rexml (3.2.5) + rspec (3.11.0) + rspec-core (~> 3.11.0) + rspec-expectations (~> 3.11.0) + rspec-mocks (~> 3.11.0) + rspec-core (3.11.0) + rspec-support (~> 3.11.0) + rspec-expectations (3.11.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.11.0) + rspec-mocks (3.11.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.11.0) + rspec-support (3.11.0) rubocop (1.29.0) parallel (~> 1.10) parser (>= 3.1.0.0) @@ -38,19 +52,23 @@ GEM ruby-progressbar (1.11.0) toml (0.3.0) parslet (>= 1.8.0, < 3.0.0) + toxiproxy (2.0.1) tzinfo (2.0.4) concurrent-ruby (~> 1.0) unicode-display_width (2.1.0) PLATFORMS + aarch64-linux arm64-darwin-21 x86_64-linux DEPENDENCIES activerecord pg + rspec rubocop - toml (~> 0.3.0) + toml + toxiproxy BUNDLED WITH - 2.3.7 + 2.3.21 diff --git a/tests/ruby/helpers/pg_instance.rb b/tests/ruby/helpers/pg_instance.rb new file mode 100644 index 000000000..311645751 --- /dev/null +++ b/tests/ruby/helpers/pg_instance.rb @@ -0,0 +1,82 @@ +require 'pg' +require 'toxiproxy' + +class PgInstance + attr_reader :port + attr_reader :username + attr_reader :password + attr_reader :database_name + + def initialize(port, username, password, database_name) + @original_port = port + @toxiproxy_port = 10000 + port.to_i + @port = @toxiproxy_port + + @username = username + @password = password + @database_name = database_name + @toxiproxy_name = "database_#{@original_port}" + Toxiproxy.populate([{ + name: @toxiproxy_name, + listen: "0.0.0.0:#{@toxiproxy_port}", + upstream: "localhost:#{@original_port}", + }]) + + # Toxiproxy server will outlive our PgInstance objects + # so we want to destroy our proxies before exiting + # Ruby finalizer is ideal for doing this + ObjectSpace.define_finalizer(@toxiproxy_name, proc { Toxiproxy[@toxiproxy_name].destroy }) + end + + def with_connection + conn = PG.connect("postgres://#{@username}:#{@password}@localhost:#{port}/#{database_name}") + yield conn + ensure + conn&.close + end + + def reset + reset_toxics + reset_stats + end + + def toxiproxy + Toxiproxy[@toxiproxy_name] + end + + def take_down + if block_given? + Toxiproxy[@toxiproxy_name].toxic(:limit_data, bytes: 5).apply { yield } + else + Toxiproxy[@toxiproxy_name].toxic(:limit_data, bytes: 5).toxics.each(&:save) + end + end + + def add_latency(latency) + if block_given? + Toxiproxy[@toxiproxy_name].toxic(:latency, latency: latency).apply { yield } + else + Toxiproxy[@toxiproxy_name].toxic(:latency, latency: latency).toxics.each(&:save) + end + end + + def delete_proxy + Toxiproxy[@toxiproxy_name].delete + end + + def reset_toxics + Toxiproxy[@toxiproxy_name].toxics.each(&:destroy) + end + + def reset_stats + with_connection { |c| c.async_exec("SELECT pg_stat_statements_reset()") } + end + + def count_query(query) + with_connection { |c| c.async_exec("SELECT SUM(calls) FROM pg_stat_statements WHERE query = '#{query}'")[0]["sum"].to_i } + end + + def count_select_1_plus_2 + with_connection { |c| c.async_exec("SELECT SUM(calls) FROM pg_stat_statements WHERE query = 'SELECT $1 + $2'")[0]["sum"].to_i } + end +end diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb new file mode 100644 index 000000000..30b2bc82e --- /dev/null +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -0,0 +1,100 @@ +require 'json' +require 'ostruct' +require_relative 'pgcat_process' +require_relative 'pg_instance' + +module Helpers + module Pgcat + def self.three_shard_setup(pool_name, pool_size) + user = { + "password" => "sharding_user", + "pool_size" => pool_size, + "statement_timeout" => 0, + "username" => "sharding_user" + } + + pgcat = PgcatProcess.new("info") + primary0 = PgInstance.new(5432, user["username"], user["password"], "shard0") + primary1 = PgInstance.new(7432, user["username"], user["password"], "shard1") + primary2 = PgInstance.new(8432, user["username"], user["password"], "shard2") + + pgcat_cfg = pgcat.current_config + pgcat_cfg["pools"] = { + "#{pool_name}" => { + "default_role" => "any", + "pool_mode" => "transaction", + "primary_reads_enabled" => false, + "query_parser_enabled" => false, + "sharding_function" => "pg_bigint_hash", + "shards" => { + "0" => { "database" => "shard0", "servers" => [["localhost", primary0.port.to_s, "primary"]] }, + "1" => { "database" => "shard1", "servers" => [["localhost", primary1.port.to_s, "primary"]] }, + "2" => { "database" => "shard2", "servers" => [["localhost", primary2.port.to_s, "primary"]] }, + }, + "users" => { "0" => user } + } + } + pgcat.update_config(pgcat_cfg) + + pgcat.start + pgcat.wait_until_ready + + OpenStruct.new.tap do |struct| + struct.pgcat = pgcat + struct.shards = [primary0, primary1, primary2] + struct.all_databases = [primary0, primary1, primary2] + end + end + + def self.single_shard_setup(pool_name, pool_size) + user = { + "password" => "sharding_user", + "pool_size" => pool_size, + "statement_timeout" => 0, + "username" => "sharding_user" + } + + pgcat = PgcatProcess.new("info") + pgcat_cfg = pgcat.current_config + + primary = PgInstance.new(5432, user["username"], user["password"], "shard0") + replica0 = PgInstance.new(7432, user["username"], user["password"], "shard0") + replica1 = PgInstance.new(8432, user["username"], user["password"], "shard0") + replica2 = PgInstance.new(9432, user["username"], user["password"], "shard0") + + # Main proxy configs + pgcat_cfg["pools"] = { + "#{pool_name}" => { + "default_role" => "any", + "pool_mode" => "transaction", + "primary_reads_enabled" => false, + "query_parser_enabled" => false, + "sharding_function" => "pg_bigint_hash", + "shards" => { + "0" => { + "database" => "shard0", + "servers" => [ + ["localhost", primary.port.to_s, "primary"], + ["localhost", replica0.port.to_s, "replica"], + ["localhost", replica1.port.to_s, "replica"], + ["localhost", replica2.port.to_s, "replica"] + ] + }, + }, + "users" => { "0" => user } + } + } + pgcat_cfg["general"]["port"] = pgcat.port + pgcat.update_config(pgcat_cfg) + pgcat.start + pgcat.wait_until_ready + + OpenStruct.new.tap do |struct| + struct.pgcat = pgcat + struct.primary = primary + struct.replicas = [replica0, replica1, replica2] + struct.all_databases = [primary, replica0, replica1, replica2] + end + end + end +end diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb new file mode 100644 index 000000000..a5a6d3d38 --- /dev/null +++ b/tests/ruby/helpers/pgcat_process.rb @@ -0,0 +1,116 @@ +require 'pg' +require 'toml' +require 'fileutils' +require 'securerandom' + +class PgcatProcess + attr_reader :port + attr_reader :pid + + def self.finalize(pid, log_filename, config_filename) + `kill #{pid}` + File.delete(config_filename) if File.exist?(config_filename) + File.delete(log_filename) if File.exist?(log_filename) + end + + def initialize(log_level) + @env = {"RUST_LOG" => log_level} + @port = rand(20000..32760) + @log_level = log_level + @log_filename = "/tmp/pgcat_log_#{SecureRandom.urlsafe_base64}.log" + @config_filename = "/tmp/pgcat_cfg_#{SecureRandom.urlsafe_base64}.toml" + + @command = "../../target/debug/pgcat #{@config_filename}" + + FileUtils.cp("../../pgcat.toml", @config_filename) + cfg = current_config + cfg["general"]["port"] = @port.to_i + cfg["general"]["enable_prometheus_exporter"] = false + + update_config(cfg) + end + + def logs + File.read(@log_filename) + end + + def update_config(config_hash) + @original_config = current_config + output_to_write = TOML::Generator.new(config_hash).body + output_to_write = output_to_write.gsub(/,\s*["|'](\d+)["|']\s*,/, ',\1,') + File.write(@config_filename, output_to_write) + end + + def current_config + old_cfg = File.read(@config_filename) + loadable_string = old_cfg.gsub(/,\s*(\d+)\s*,/, ', "\1",') + TOML.load(loadable_string) + end + + def reload_config + `kill -s HUP #{@pid}` + sleep 0.1 + end + + def start + raise StandardError, "Process is already started" unless @pid.nil? + @pid = Process.spawn(@env, @command, err: @log_filename, out: @log_filename) + ObjectSpace.define_finalizer(@log_filename, proc { PgcatProcess.finalize(@pid, @log_filename, @config_filename) }) + + return self + end + + def wait_until_ready + exc = nil + 10.times do + PG::connect(example_connection_string).close + + return self + rescue => e + exc = e + sleep(0.5) + end + puts exc + raise StandardError, "Process #{@pid} never became ready. Logs #{logs}" + end + + def stop + `kill #{@pid}` + sleep 0.1 + end + + def shutdown + stop + File.delete(@config_filename) if File.exist?(@config_filename) + File.delete(@log_filename) if File.exist?(@log_filename) + end + + def admin_connection_string + cfg = current_config + username = cfg["general"]["admin_username"] + password = cfg["general"]["admin_password"] + + "postgresql://#{username}:#{password}@0.0.0.0:#{@port}/pgcat" + end + + def connection_string(pool_name, username) + cfg = current_config + + user_idx, user_obj = cfg["pools"][pool_name]["users"].detect { |k, user| user["username"] == username } + password = user_obj["password"] + + "postgresql://#{username}:#{password}@0.0.0.0:#{@port}/#{pool_name}" + end + + def example_connection_string + cfg = current_config + first_pool_name = cfg["pools"].keys[0] + + db_name = first_pool_name + + username = cfg["pools"][first_pool_name]["users"]["0"]["username"] + password = cfg["pools"][first_pool_name]["users"]["0"]["password"] + + "postgresql://#{username}:#{password}@0.0.0.0:#{@port}/#{db_name}" + end +end diff --git a/tests/ruby/load_balancing_spec.rb b/tests/ruby/load_balancing_spec.rb new file mode 100644 index 000000000..bd98a831c --- /dev/null +++ b/tests/ruby/load_balancing_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true +require_relative 'spec_helper' + +describe "Load Balancing" do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5) } + after do + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + + context "under regular circumstances" do + it "balances query volume between all instances" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + + query_count = QUERY_COUNT + expected_share = query_count / processes.all_databases.count + failed_count = 0 + + query_count.times do + conn.async_exec("SELECT 1 + 2") + rescue + failed_count += 1 + end + + expect(failed_count).to eq(0) + processes.all_databases.map(&:count_select_1_plus_2).each do |instance_share| + expect(instance_share).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share) + end + end + end + + context "when some replicas are down" do + it "balances query volume between working instances" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + expected_share = QUERY_COUNT / (processes.all_databases.count - 2) + failed_count = 0 + + processes[:replicas][0].take_down do + processes[:replicas][1].take_down do + QUERY_COUNT.times do + conn.async_exec("SELECT 1 + 2") + rescue + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + failed_count += 1 + end + end + end + + expect(failed_count).to eq(2) + processes.all_databases.each do |instance| + queries_routed = instance.count_select_1_plus_2 + if processes.replicas[0..1].include?(instance) + expect(queries_routed).to eq(0) + else + expect(queries_routed).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share) + end + end + end + end +end + diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb new file mode 100644 index 000000000..9aee49afe --- /dev/null +++ b/tests/ruby/misc_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true +require_relative 'spec_helper' + +describe "Miscellaneous" do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5) } + after do + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + + describe "Extended Protocol handling" do + it "does not send packets that client does not expect during extended protocol sequence" do + new_configs = processes.pgcat.current_config + + new_configs["general"]["connect_timeout"] = 500 + new_configs["general"]["ban_time"] = 1 + new_configs["general"]["shutdown_timeout"] = 1 + new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 + + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + + 25.times do + Thread.new do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SELECT pg_sleep(5)") rescue PG::SystemError + ensure + conn&.close + end + end + + sleep(0.5) + conn_under_test = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + stdout, stderr = with_captured_stdout_stderr do + 15.times do |i| + conn_under_test.async_exec("SELECT 1") rescue PG::SystemError + conn_under_test.exec_params("SELECT #{i} + $1", [i]) rescue PG::SystemError + sleep 1 + end + end + + raise StandardError, "Libpq got unexpected messages while idle" if stderr.include?("arrived from server while idle") + end + end + + describe "Pool recycling after config reload" do + let(:processes) { Helpers::Pgcat.three_shard_setup("sharded_db", 5) } + + it "should update pools for new clients and clients that are no longer in transaction" do + server_conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + server_conn.async_exec("BEGIN") + + # No config change yet, client should set old configs + current_datebase_from_pg = server_conn.async_exec("SELECT current_database();")[0]["current_database"] + expect(current_datebase_from_pg).to eq('shard0') + + # Swap shards + new_config = processes.pgcat.current_config + shard0 = new_config["pools"]["sharded_db"]["shards"]["0"] + shard1 = new_config["pools"]["sharded_db"]["shards"]["1"] + new_config["pools"]["sharded_db"]["shards"]["0"] = shard1 + new_config["pools"]["sharded_db"]["shards"]["1"] = shard0 + + # Reload config + processes.pgcat.update_config(new_config) + processes.pgcat.reload_config + sleep 0.5 + + # Config changed but transaction is in progress, client should set old configs + current_datebase_from_pg = server_conn.async_exec("SELECT current_database();")[0]["current_database"] + expect(current_datebase_from_pg).to eq('shard0') + server_conn.async_exec("COMMIT") + + # Transaction finished, client should get new configs + current_datebase_from_pg = server_conn.async_exec("SELECT current_database();")[0]["current_database"] + expect(current_datebase_from_pg).to eq('shard1') + + # New connection should get new configs + server_conn.close() + server_conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + current_datebase_from_pg = server_conn.async_exec("SELECT current_database();")[0]["current_database"] + expect(current_datebase_from_pg).to eq('shard1') + end + end + + describe "Clients closing connection in the middle of transaction" do + it "sends a rollback to the server" do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'primary'") + conn.async_exec("BEGIN") + conn.close + + expect(processes.primary.count_query("ROLLBACK")).to eq(1) + expect(processes.primary.count_query("DISCARD ALL")).to eq(1) + end + end + + describe "Server version reporting" do + it "reports correct version for normal and admin databases" do + server_conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + expect(server_conn.server_version).not_to eq(0) + server_conn.close + + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + expect(admin_conn.server_version).not_to eq(0) + admin_conn.close + end + end +end diff --git a/tests/ruby/routing_spec.rb b/tests/ruby/routing_spec.rb new file mode 100644 index 000000000..24ea137af --- /dev/null +++ b/tests/ruby/routing_spec.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true +require_relative 'spec_helper' + + +describe "Routing" do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5) } + after do + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + + describe "SET ROLE" do + context "primary" do + it "routes queries only to primary" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'primary'") + + query_count = 30 + failed_count = 0 + + query_count.times do + conn.async_exec("SELECT 1 + 2") + rescue + failed_count += 1 + end + + expect(failed_count).to eq(0) + processes.replicas.map(&:count_select_1_plus_2).each do |instance_share| + expect(instance_share).to eq(0) + end + + expect(processes.primary.count_select_1_plus_2).to eq(query_count) + end + end + context "replica" do + it "routes queries only to replicas" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'replica'") + + expected_share = QUERY_COUNT / processes.replicas.count + failed_count = 0 + + QUERY_COUNT.times do + conn.async_exec("SELECT 1 + 2") + rescue + failed_count += 1 + end + + expect(failed_count).to eq(0) + + processes.replicas.map(&:count_select_1_plus_2).each do |instance_share| + expect(instance_share).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share) + end + + expect(processes.primary.count_select_1_plus_2).to eq(0) + end + end + + context "any" do + it "routes queries to all instances" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'any'") + + expected_share = QUERY_COUNT / processes.all_databases.count + failed_count = 0 + + QUERY_COUNT.times do + conn.async_exec("SELECT 1 + 2") + rescue + failed_count += 1 + end + + expect(failed_count).to eq(0) + + processes.all_databases.map(&:count_select_1_plus_2).each do |instance_share| + expect(instance_share).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share) + end + end + end + end +end diff --git a/tests/ruby/spec_helper.rb b/tests/ruby/spec_helper.rb new file mode 100644 index 000000000..3050e18aa --- /dev/null +++ b/tests/ruby/spec_helper.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'pg' +require_relative 'helpers/pgcat_helper' + +QUERY_COUNT = 300 +MARGIN_OF_ERROR = 0.30 + +def with_captured_stdout_stderr + sout = STDOUT.clone + serr = STDERR.clone + STDOUT.reopen("/tmp/out.txt", "w+") + STDERR.reopen("/tmp/err.txt", "w+") + STDOUT.sync = true + STDERR.sync = true + yield + return File.read('/tmp/out.txt'), File.read('/tmp/err.txt') +ensure + STDOUT.reopen(sout) + STDERR.reopen(serr) +end diff --git a/tests/ruby/tests.rb b/tests/ruby/tests.rb index d7b928d4a..1c18e15fd 100644 --- a/tests/ruby/tests.rb +++ b/tests/ruby/tests.rb @@ -1,93 +1,6 @@ # frozen_string_literal: true - -require 'active_record' require 'pg' -require 'toml' - -$stdout.sync = true -$stderr.sync = true - -class ConfigEditor - def initialize - @original_config_text = File.read('../../.circleci/pgcat.toml') - text_to_load = @original_config_text.gsub("5432", "\"5432\"") - - @original_configs = TOML.load(text_to_load) - end - - def original_configs - TOML.load(TOML::Generator.new(@original_configs).body) - end - - def with_modified_configs(new_configs) - text_to_write = TOML::Generator.new(new_configs).body - text_to_write = text_to_write.gsub("\"5432\"", "5432") - File.write('../../.circleci/pgcat.toml', text_to_write) - yield - ensure - File.write('../../.circleci/pgcat.toml', @original_config_text) - end -end - -def with_captured_stdout_stderr - sout = STDOUT.clone - serr = STDERR.clone - STDOUT.reopen("/tmp/out.txt", "w+") - STDERR.reopen("/tmp/err.txt", "w+") - STDOUT.sync = true - STDERR.sync = true - yield - return File.read('/tmp/out.txt'), File.read('/tmp/err.txt') -ensure - STDOUT.reopen(sout) - STDERR.reopen(serr) -end - - -def test_extended_protocol_pooler_errors - admin_conn = PG::connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat") - - conf_editor = ConfigEditor.new - new_configs = conf_editor.original_configs - - # shorter timeouts - new_configs["general"]["connect_timeout"] = 500 - new_configs["general"]["ban_time"] = 1 - new_configs["general"]["shutdown_timeout"] = 1 - new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 - new_configs["pools"]["sharded_db"]["users"]["1"]["pool_size"] = 1 - - conf_editor.with_modified_configs(new_configs) { admin_conn.async_exec("RELOAD") } - - conn_str = "postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db" - 10.times do - Thread.new do - conn = PG::connect(conn_str) - conn.async_exec("SELECT pg_sleep(5)") rescue PG::SystemError - ensure - conn&.close - end - end - - sleep(0.5) - conn_under_test = PG::connect(conn_str) - stdout, stderr = with_captured_stdout_stderr do - 5.times do |i| - conn_under_test.async_exec("SELECT 1") rescue PG::SystemError - conn_under_test.exec_params("SELECT #{i} + $1", [i]) rescue PG::SystemError - sleep 1 - end - end - - raise StandardError, "Libpq got unexpected messages while idle" if stderr.include?("arrived from server while idle") - puts "Pool checkout errors not breaking clients passed" -ensure - sleep 1 - admin_conn.async_exec("RELOAD") # Reset state - conn_under_test&.close -end - -test_extended_protocol_pooler_errors +require 'active_record' # Uncomment these two to see all queries. # ActiveRecord.verbose_query_logs = true @@ -198,68 +111,3 @@ def down rescue ActiveRecord::StatementInvalid puts 'OK' end - -# Test evil clients -def poorly_behaved_client - conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") - conn.async_exec 'BEGIN' - conn.async_exec 'SELECT 1' - - conn.close - puts 'Bad client ok' -end - -25.times do - poorly_behaved_client -end - - -def test_server_parameters - server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") - raise StandardError, "Bad server version" if server_conn.server_version == 0 - server_conn.close - - admin_conn = PG::connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat") - raise StandardError, "Bad server version" if admin_conn.server_version == 0 - admin_conn.close - - puts 'Server parameters ok' -end - - -def test_reload_pool_recycling - admin_conn = PG::connect("postgres://admin_user:admin_pass@127.0.0.1:6432/pgcat") - server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") - - server_conn.async_exec("BEGIN") - conf_editor = ConfigEditor.new - new_configs = conf_editor.original_configs - - # swap shards - new_configs["pools"]["sharded_db"]["shards"]["0"]["database"] = "shard1" - new_configs["pools"]["sharded_db"]["shards"]["1"]["database"] = "shard0" - - raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard0' - conf_editor.with_modified_configs(new_configs) { admin_conn.async_exec("RELOAD") } - raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard0' - server_conn.async_exec("COMMIT;") - - # Transaction finished, client should get new configs - raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard1' - server_conn.close() - - # New connection should get new configs - server_conn = PG::connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db?application_name=testing_pgcat") - raise StandardError if server_conn.async_exec("SELECT current_database();")[0]["current_database"] != 'shard1' - -ensure - admin_conn.async_exec("RELOAD") # Go back to old state - admin_conn.close - server_conn.close - puts "Pool Recycling okay!" -end - -test_reload_pool_recycling - - - diff --git a/tests/sharding/query_routing_setup.sql b/tests/sharding/query_routing_setup.sql index 384b23414..c25ac18e2 100644 --- a/tests/sharding/query_routing_setup.sql +++ b/tests/sharding/query_routing_setup.sql @@ -70,23 +70,35 @@ GRANT CONNECT ON DATABASE shard2 TO other_user; GRANT CONNECT ON DATABASE some_db TO simple_user; \c shard0 +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; +GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO sharding_user; GRANT ALL ON SCHEMA public TO sharding_user; GRANT ALL ON TABLE data TO sharding_user; GRANT ALL ON SCHEMA public TO other_user; GRANT ALL ON TABLE data TO other_user; +GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO other_user; \c shard1 +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; +GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO sharding_user; GRANT ALL ON SCHEMA public TO sharding_user; GRANT ALL ON TABLE data TO sharding_user; GRANT ALL ON SCHEMA public TO other_user; GRANT ALL ON TABLE data TO other_user; +GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO other_user; + \c shard2 +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; +GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO sharding_user; GRANT ALL ON SCHEMA public TO sharding_user; GRANT ALL ON TABLE data TO sharding_user; GRANT ALL ON SCHEMA public TO other_user; GRANT ALL ON TABLE data TO other_user; +GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO other_user; \c some_db +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; +GRANT EXECUTE ON FUNCTION pg_stat_statements_reset TO simple_user; GRANT ALL ON SCHEMA public TO simple_user; GRANT ALL ON TABLE data TO simple_user; From 20ec5af608cbf44e6489ec9335c99274fec37218 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 30 Aug 2022 12:43:45 -0500 Subject: [PATCH 196/551] Allow running integration tests with coverage locally (#151) --- README.md | 16 ++++++++--- tests/docker/Dockerfile | 5 ++++ tests/docker/docker-compose.yml | 47 +++++++++++++++++++++++++++++++++ tests/docker/run.sh | 21 +++++++++++++++ 4 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 tests/docker/Dockerfile create mode 100644 tests/docker/docker-compose.yml create mode 100644 tests/docker/run.sh diff --git a/README.md b/README.md index 48f835348..8d4c77fa1 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,14 @@ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended See [sharding README](./tests/sharding/README.md) for sharding logic testing. +Run `cargo test` to run Rust tests. + +Run the following commands to run Integration tests locally. +``` +cd tests/docker/ +docker compose up --exit-code-from main # This will also produce coverage report under ./cov/ +``` + | **Feature** | **Tested in CI** | **Tested manually** | **Comments** | |-----------------------|--------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------| | Transaction pooling | :white_check_mark: | :white_check_mark: | Used by default for all tests. | @@ -447,7 +455,7 @@ Always good to have a base line. ``` $ pgbench -t 1000 -c 16 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: +Password: starting vacuum...end. transaction type: scaling factor: 1 @@ -461,7 +469,7 @@ tps = 139443.955722 (including connections establishing) tps = 142314.859075 (excluding connections establishing) $ pgbench -t 1000 -c 32 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: +Password: starting vacuum...end. transaction type: scaling factor: 1 @@ -475,7 +483,7 @@ tps = 150644.840891 (including connections establishing) tps = 152218.499430 (excluding connections establishing) $ pgbench -t 1000 -c 64 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: +Password: starting vacuum...end. transaction type: scaling factor: 1 @@ -489,7 +497,7 @@ tps = 152517.663404 (including connections establishing) tps = 153319.188482 (excluding connections establishing) $ pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: +Password: starting vacuum...end. transaction type: scaling factor: 1 diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile new file mode 100644 index 000000000..1f11efb3e --- /dev/null +++ b/tests/docker/Dockerfile @@ -0,0 +1,5 @@ +FROM rust:bullseye + +RUN apt-get update && apt-get install llvm-11 psmisc postgresql-contrib postgresql-client ruby ruby-dev libpq-dev python3 python3-pip lcov sudo curl -y +RUN cargo install cargo-binutils rustfilt +RUN rustup component add llvm-tools-preview diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml new file mode 100644 index 000000000..d86e23990 --- /dev/null +++ b/tests/docker/docker-compose.yml @@ -0,0 +1,47 @@ +version: "3" +services: + pg1: + image: postgres:14 + network_mode: "service:main" + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "5432"] + pg2: + image: postgres:14 + network_mode: "service:main" + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "7432"] + pg3: + image: postgres:14 + network_mode: "service:main" + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "8432"] + pg4: + image: postgres:14 + network_mode: "service:main" + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "9432"] + main: + build: . + command: ["bash", "/app/tests/docker/run.sh"] + environment: + RUSTFLAGS: "-C instrument-coverage" + LLVM_PROFILE_FILE: "pgcat-%m.profraw" + volumes: + - ../../:/app/ + - /app/target/ diff --git a/tests/docker/run.sh b/tests/docker/run.sh new file mode 100644 index 000000000..ada5d9ee5 --- /dev/null +++ b/tests/docker/run.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +rm /app/*.profraw || true +rm /app/pgcat.profdata || true +rm -rf /app/cov || true + +cd /app/ + +cargo build +cargo test --tests + +bash .circleci/run_tests.sh + +rust-profdata merge -sparse pgcat-*.profraw -o pgcat.profdata + +rust-cov export -ignore-filename-regex="rustc|registry" -Xdemangler=rustfilt -instr-profile=pgcat.profdata --object ./target/debug/pgcat --format lcov > ./lcov.info + +genhtml lcov.info --output-directory cov --prefix $(pwd) + +rm /app/*.profraw +rm /app/pgcat.profdata From 743823e4e65a47799b1b2d22b465b35d588a8212 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 1 Sep 2022 13:16:22 -0500 Subject: [PATCH 197/551] Log Address information in connection create/drop (#154) * Log Address information in connection create/drop * run ci --- src/pool.rs | 6 +----- src/server.rs | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pool.rs b/src/pool.rs index 99cccaf10..dea29ad24 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -559,11 +559,7 @@ impl ManageConnection for ServerPool { /// Attempts to create a new connection. async fn connect(&self) -> Result { - info!( - "Creating a new connection to {:?} using user {:?}", - self.address.name(), - self.user.username - ); + info!("Creating a new server connection {:?}", self.address); // Put a temporary process_id into the stats // for server login. diff --git a/src/server.rs b/src/server.rs index 3134a65df..65ca6c623 100644 --- a/src/server.rs +++ b/src/server.rs @@ -607,7 +607,8 @@ impl Drop for Server { let duration = now - self.connected_at; info!( - "Server connection closed, session duration: {}", + "Server connection closed {:?}, session duration: {}", + self.address, crate::format_duration(&duration) ); } From 4bd8321746fabab9c21256f08a461aa51075c89e Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 1 Sep 2022 17:02:39 -0500 Subject: [PATCH 198/551] Better handling extended protocol messages in the event of busy pool (#155) * Better handling for checkout errors during extended protocol messages * Fix specs * comment --- src/client.rs | 50 +++++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/client.rs b/src/client.rs index 419448fb8..67d916aa8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -600,12 +600,6 @@ where message_result = read_message(&mut self.read) => message_result? }; - // Avoid taking a server if the client just wants to disconnect. - if message[0] as char == 'X' { - debug!("Client disconnecting"); - return Ok(()); - } - // Handle admin database queries. if self.admin { debug!("Handling admin command"); @@ -613,6 +607,25 @@ where continue; } + match message[0] as char { + // Buffer extended protocol messages even if we do not have + // a server connection yet. Hopefully, when we get the S message + // we'll be able to allocate a connection. Also, clients do not expect + // the server to respond to these messages so even if we were not able to + // allocate a connection, we wouldn't be able to send back an error message + // to the client so we buffer them and defer the decision to error out or not + // to when we get the S message + 'P' | 'B' | 'D' | 'E' => { + self.buffer.put(&message[..]); + continue; + } + 'X' => { + debug!("Client disconnecting"); + return Ok(()); + } + _ => (), + } + // Get a pool instance referenced by the most up-to-date // pointer. This ensures we always read the latest config // when starting a query. @@ -714,22 +727,17 @@ where conn } Err(err) => { - // Clients do not expect to get SystemError followed by ReadyForQuery in the middle - // of extended protocol submission. So we will hold off on sending the actual error - // message to the client until we get 'S' message - match message[0] as char { - 'P' | 'B' | 'E' | 'D' => (), - _ => { - error_response( - &mut self.write, - "could not get connection from the pool", - ) - .await?; - } - }; - + // Client is attempting to get results from the server, + // but we were unable to grab a connection from the pool + // We'll send back an error message and clean the extended + // protocol buffer + if message[0] as char == 'S' { + error!("Got Sync message but failed to get a connection from the pool"); + self.buffer.clear(); + } + error_response(&mut self.write, "could not get connection from the pool") + .await?; error!("Could not get connection from pool: {:?}", err); - continue; } }; From 99af93e00bf8fe6c25a2911295c69ecec26ec26e Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 1 Sep 2022 22:06:55 -0500 Subject: [PATCH 199/551] Send DISCARD ALL even if client is not in transaction (#152) * Send DISCARD ALL even if client is not in transaction * fmt * Added tests + avoided sending extra discard all * Adds set name logic to beginning of handle client * fmt * refactor dead code handling * Refactor reading command tag * remove unnecessary trim * Removing debugging statement * typo * typo{ * documentation * edit text * un-unwrap * run ci * run ci --- .circleci/run_tests.sh | 6 +-- src/client.rs | 40 +++++++-------- src/server.rs | 65 ++++++++++++++++++++++-- tests/ruby/helpers/pgcat_helper.rb | 8 +-- tests/ruby/misc_spec.rb | 79 +++++++++++++++++++++++++++++- 5 files changed, 164 insertions(+), 34 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 1585ebd8e..6ffef8bae 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -90,8 +90,8 @@ kill -SIGHUP $(pgrep pgcat) # Reload config again cd tests/ruby sudo gem install bundler bundle install -bundle exec ruby tests.rb -bundle exec rspec *_spec.rb +bundle exec ruby tests.rb || exit 1 +bundle exec rspec *_spec.rb || exit 1 cd ../.. # @@ -99,7 +99,7 @@ cd ../.. # These tests will start and stop the pgcat server so it will need to be restarted after the tests # pip3 install -r tests/python/requirements.txt -python3 tests/python/tests.py +python3 tests/python/tests.py || exit 1 start_pgcat "info" diff --git a/src/client.rs b/src/client.rs index 67d916aa8..3ce4afb92 100644 --- a/src/client.rs +++ b/src/client.rs @@ -59,6 +59,7 @@ pub struct Client { client_server_map: ClientServerMap, /// Client parameters, e.g. user, client_encoding, etc. + #[allow(dead_code)] parameters: HashMap, /// Statistics @@ -82,6 +83,9 @@ pub struct Client { /// Postgres user for this client (This comes from the user in the connection string) username: String, + /// Application name for this client (defaults to pgcat) + application_name: String, + /// Used to notify clients about an impending shutdown shutdown: Receiver<()>, } @@ -365,6 +369,11 @@ where None => return Err(Error::ClientError), }; + let application_name = match parameters.get("application_name") { + Some(application_name) => application_name, + None => "pgcat", + }; + let admin = ["pgcat", "pgbouncer"] .iter() .filter(|db| *db == &pool_name) @@ -493,6 +502,7 @@ where last_server_id: None, pool_name: pool_name.clone(), username: username.clone(), + application_name: application_name.to_string(), shutdown, connected_to_server: false, }); @@ -526,6 +536,7 @@ where last_server_id: None, pool_name: String::from("undefined"), username: String::from("undefined"), + application_name: String::from("undefined"), shutdown, connected_to_server: false, }); @@ -767,13 +778,10 @@ where server.address() ); - // Set application_name if any. // TODO: investigate other parameters and set them too. - if self.parameters.contains_key("application_name") { - server - .set_name(&self.parameters["application_name"]) - .await?; - } + + // Set application_name. + server.set_name(&self.application_name).await?; // Transaction loop. Multiple queries can be issued by the client here. // The connection belongs to the client until the transaction is over, @@ -790,12 +798,7 @@ where Err(err) => { // Client disconnected inside a transaction. // Clean up the server and re-use it. - // This prevents connection thrashing by bad clients. - if server.in_transaction() { - server.query("ROLLBACK").await?; - server.query("DISCARD ALL").await?; - server.set_name("pgcat").await?; - } + server.checkin_cleanup().await?; return Err(err); } @@ -837,16 +840,7 @@ where // Terminate 'X' => { - // Client closing. Rollback and clean up - // connection before releasing into the pool. - // Pgbouncer closes the connection which leads to - // connection thrashing when clients misbehave. - if server.in_transaction() { - server.query("ROLLBACK").await?; - server.query("DISCARD ALL").await?; - server.set_name("pgcat").await?; - } - + server.checkin_cleanup().await?; self.release(); return Ok(()); @@ -950,8 +944,10 @@ where // The server is no longer bound to us, we can't cancel it's queries anymore. debug!("Releasing server back into the pool"); + server.checkin_cleanup().await?; self.stats.server_idle(server.process_id(), address.id); self.connected_to_server = false; + self.release(); self.stats.client_idle(self.process_id, address.id); } diff --git a/src/server.rs b/src/server.rs index 65ca6c623..e04676ec7 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,7 +1,8 @@ /// Implementation of the PostgreSQL server (database) protocol. /// Here we are pretending to the a Postgres client. use bytes::{Buf, BufMut, BytesMut}; -use log::{debug, error, info, trace}; +use log::{debug, error, info, trace, warn}; +use std::io::Read; use std::time::SystemTime; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ @@ -48,6 +49,9 @@ pub struct Server { /// Is the server broken? We'll remote it from the pool if so. bad: bool, + /// If server connection requires a DISCARD ALL before checkin + needs_cleanup: bool, + /// Mapping of clients and servers used for query cancellation. client_server_map: ClientServerMap, @@ -316,6 +320,7 @@ impl Server { in_transaction: false, data_available: false, bad: false, + needs_cleanup: false, client_server_map: client_server_map, connected_at: chrono::offset::Utc::now().naive_utc(), stats: stats, @@ -440,6 +445,29 @@ impl Server { break; } + // CommandComplete + 'C' => { + let mut command_tag = String::new(); + match message.reader().read_to_string(&mut command_tag) { + Ok(_) => { + // Non-exhaustive list of commands that are likely to change session variables/resources + // which can leak between clients. This is a best effort to block bad clients + // from poisoning a transaction-mode pool by setting inappropriate session variables + match command_tag.as_str() { + "SET\0" | "PREPARE\0" => { + debug!("Server connection marked for clean up"); + self.needs_cleanup = true; + } + _ => (), + } + } + + Err(err) => { + warn!("Encountered an error while parsing CommandTag {}", err); + } + } + } + // DataRow 'D' => { // More data is available after this message, this is not the end of the reply. @@ -553,14 +581,43 @@ impl Server { Ok(()) } + /// Perform any necessary cleanup before putting the server + /// connection back in the pool + pub async fn checkin_cleanup(&mut self) -> Result<(), Error> { + // Client disconnected with an open transaction on the server connection. + // Pgbouncer behavior is to close the server connection but that can cause + // server connection thrashing if clients repeatedly do this. + // Instead, we ROLLBACK that transaction before putting the connection back in the pool + if self.in_transaction() { + self.query("ROLLBACK").await?; + } + + // Client disconnected but it perfromed session-altering operations such as + // SET statement_timeout to 1 or create a prepared statement. We clear that + // to avoid leaking state between clients. For performance reasons we only + // send `DISCARD ALL` if we think the session is altered instead of just sending + // it before each checkin. + if self.needs_cleanup { + self.query("DISCARD ALL").await?; + self.needs_cleanup = false; + } + + return Ok(()); + } + /// A shorthand for `SET application_name = $1`. - #[allow(dead_code)] pub async fn set_name(&mut self, name: &str) -> Result<(), Error> { if self.application_name != name { self.application_name = name.to_string(); - Ok(self + // We don't want `SET application_name` to mark the server connection + // as needing cleanup + let needs_cleanup_before = self.needs_cleanup; + + let result = Ok(self .query(&format!("SET application_name = '{}'", name)) - .await?) + .await?); + self.needs_cleanup = needs_cleanup_before; + return result; } else { Ok(()) } diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index 30b2bc82e..80ac9dab6 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -5,7 +5,7 @@ module Helpers module Pgcat - def self.three_shard_setup(pool_name, pool_size) + def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction") user = { "password" => "sharding_user", "pool_size" => pool_size, @@ -22,7 +22,7 @@ def self.three_shard_setup(pool_name, pool_size) pgcat_cfg["pools"] = { "#{pool_name}" => { "default_role" => "any", - "pool_mode" => "transaction", + "pool_mode" => pool_mode, "primary_reads_enabled" => false, "query_parser_enabled" => false, "sharding_function" => "pg_bigint_hash", @@ -46,7 +46,7 @@ def self.three_shard_setup(pool_name, pool_size) end end - def self.single_shard_setup(pool_name, pool_size) + def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction") user = { "password" => "sharding_user", "pool_size" => pool_size, @@ -66,7 +66,7 @@ def self.single_shard_setup(pool_name, pool_size) pgcat_cfg["pools"] = { "#{pool_name}" => { "default_role" => "any", - "pool_mode" => "transaction", + "pool_mode" => pool_mode, "primary_reads_enabled" => false, "query_parser_enabled" => false, "sharding_function" => "pg_bigint_hash", diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index 9aee49afe..d5b529a9e 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -91,7 +91,6 @@ conn.close expect(processes.primary.count_query("ROLLBACK")).to eq(1) - expect(processes.primary.count_query("DISCARD ALL")).to eq(1) end end @@ -106,4 +105,82 @@ admin_conn.close end end + + describe "State clearance" do + context "session mode" do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5, "session") } + + it "Clears state before connection checkin" do + # Both modes of operation should not raise + # ERROR: prepared statement "prepared_q" already exists + 15.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("PREPARE prepared_q (int) AS SELECT $1") + conn.close + end + + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + initial_value = conn.async_exec("SHOW statement_timeout")[0]["statement_timeout"] + conn.async_exec("SET statement_timeout to 1000") + current_value = conn.async_exec("SHOW statement_timeout")[0]["statement_timeout"] + expect(conn.async_exec("SHOW statement_timeout")[0]["statement_timeout"]).to eq("1s") + conn.close + end + + it "Does not send DISCARD ALL unless necessary" do + 10.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'primary'") + conn.async_exec("SELECT 1") + conn.close + end + + expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + + 10.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'primary'") + conn.async_exec("SELECT 1") + conn.async_exec("SET statement_timeout to 5000") + conn.close + end + + expect(processes.primary.count_query("DISCARD ALL")).to eq(10) + end + end + + context "transaction mode" do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5, "transaction") } + it "Clears state before connection checkin" do + # Both modes of operation should not raise + # ERROR: prepared statement "prepared_q" already exists + 15.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("PREPARE prepared_q (int) AS SELECT $1") + conn.close + end + end + + it "Does not send DISCARD ALL unless necessary" do + 10.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'primary'") + conn.async_exec("SELECT 1") + conn.close + end + + expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + + 10.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'primary'") + conn.async_exec("SELECT 1") + conn.async_exec("SET statement_timeout to 5000") + conn.close + end + + expect(processes.primary.count_query("DISCARD ALL")).to eq(10) + end + end + end end From 6b716b705f4123488d6eca94e7da1c7227e04878 Mon Sep 17 00:00:00 2001 From: zain-kabani Date: Mon, 5 Sep 2022 04:02:49 -0400 Subject: [PATCH 200/551] Patch graceful shutdown bug (#157) * Fixes non-admin client counting error * Add log when sigterm received and log number of active clients when shutdown timeout is reached --- src/client.rs | 4 ++-- src/main.rs | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/client.rs b/src/client.rs index 3ce4afb92..2ddafa0d8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -204,7 +204,7 @@ pub async fn client_entrypoint( Ok(mut client) => { info!("Client {:?} connected (plain)", addr); - if client.is_admin() { + if !client.is_admin() { let _ = drain.send(1).await; } @@ -229,7 +229,7 @@ pub async fn client_entrypoint( Ok(mut client) => { info!("Client {:?} issued a cancel query request", addr); - if client.is_admin() { + if !client.is_admin() { let _ = drain.send(1).await; } diff --git a/src/main.rs b/src/main.rs index a0c1d7cf4..781504019 100644 --- a/src/main.rs +++ b/src/main.rs @@ -221,13 +221,16 @@ async fn main() { interval.tick().await; // We're done waiting. - error!("Timed out waiting for clients"); + error!("Graceful shutdown timed out. {} active clients being closed", total_clients); let _ = exit_tx.send(()).await; }); }, - _ = term_signal.recv() => break, + _ = term_signal.recv() => { + info!("Got SIGTERM, closing with {} clients active", total_clients); + break; + }, new_client = listener.accept() => { let (socket, addr) = match new_client { From 0a4a7391aa556e31e386bcf4172cb5a7389bd5e1 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 5 Sep 2022 03:21:06 -0500 Subject: [PATCH 201/551] Main Thread Panic when swarmed with clients (#158) * Main Thread Panic when swarmed with clients * fix * fix * 1024 * fix * remove test * Update src/client.rs * Update src/main.rs * Update src/client.rs * Update src/main.rs --- src/client.rs | 2 +- src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client.rs b/src/client.rs index 2ddafa0d8..ffda6f104 100644 --- a/src/client.rs +++ b/src/client.rs @@ -95,7 +95,7 @@ pub async fn client_entrypoint( mut stream: TcpStream, client_server_map: ClientServerMap, shutdown: Receiver<()>, - drain: Sender, + drain: Sender, admin_only: bool, ) -> Result<(), Error> { // Figure out if the client wants TLS or not. diff --git a/src/main.rs b/src/main.rs index 781504019..a6075efdb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -178,7 +178,7 @@ async fn main() { let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); let mut sighup_signal = unix_signal(SignalKind::hangup()).unwrap(); let (shutdown_tx, _) = broadcast::channel::<()>(1); - let (drain_tx, mut drain_rx) = mpsc::channel::(2048); + let (drain_tx, mut drain_rx) = mpsc::channel::(2048); let (exit_tx, mut exit_rx) = mpsc::channel::<()>(1); info!("Waiting for clients"); From 9d8a4875bdc410b481f2777dfb94f34ddfe03978 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Mon, 5 Sep 2022 04:21:27 -0400 Subject: [PATCH 202/551] Adds microsecond logging and also reformats duration to include milliseconds (#156) * Adds microsecond logging and also reformats duration to include milliseconds * fmt * attempt to fix cd * revert --- src/main.rs | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/src/main.rs b/src/main.rs index a6075efdb..0d4bd37ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -74,7 +74,8 @@ use crate::stats::{Collector, Reporter, REPORTER}; #[tokio::main(worker_threads = 4)] async fn main() { - env_logger::init(); + env_logger::builder().format_timestamp_micros().init(); + info!("Welcome to PgCat! Meow. (Version {})", VERSION); if !query_router::QueryRouter::setup() { @@ -307,34 +308,18 @@ async fn main() { /// /// * `duration` - A duration of time fn format_duration(duration: &chrono::Duration) -> String { - let seconds = { - let seconds = duration.num_seconds() % 60; - if seconds < 10 { - format!("0{}", seconds) - } else { - format!("{}", seconds) - } - }; + let milliseconds = format!("{:0>3}", duration.num_milliseconds() % 1000); - let minutes = { - let minutes = duration.num_minutes() % 60; - if minutes < 10 { - format!("0{}", minutes) - } else { - format!("{}", minutes) - } - }; + let seconds = format!("{:0>2}", duration.num_seconds() % 60); - let hours = { - let hours = duration.num_hours() % 24; - if hours < 10 { - format!("0{}", hours) - } else { - format!("{}", hours) - } - }; + let minutes = format!("{:0>2}", duration.num_minutes() % 60); + + let hours = format!("{:0>2}", duration.num_hours() % 24); let days = duration.num_days().to_string(); - format!("{}d {}:{}:{}", days, hours, minutes, seconds) + format!( + "{}d {}:{}:{}.{}", + days, hours, minutes, seconds, milliseconds + ) } From 71206f75ec805f592ce2270770dfd1dc40175094 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 7 Sep 2022 00:22:31 -0500 Subject: [PATCH 203/551] Avoid reporting ProtocolSyncError when admin session disconnects (#160) * Avoid reporting ProtocolSyncError when admin session disconnects * rebuild * rebuild --- src/client.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/client.rs b/src/client.rs index ffda6f104..7fe633c08 100644 --- a/src/client.rs +++ b/src/client.rs @@ -611,13 +611,6 @@ where message_result = read_message(&mut self.read) => message_result? }; - // Handle admin database queries. - if self.admin { - debug!("Handling admin command"); - handle_admin(&mut self.write, message, self.client_server_map.clone()).await?; - continue; - } - match message[0] as char { // Buffer extended protocol messages even if we do not have // a server connection yet. Hopefully, when we get the S message @@ -637,6 +630,13 @@ where _ => (), } + // Handle admin database queries. + if self.admin { + debug!("Handling admin command"); + handle_admin(&mut self.write, message, self.client_server_map.clone()).await?; + continue; + } + // Get a pool instance referenced by the most up-to-date // pointer. This ensures we always read the latest config // when starting a query. From eda559f2ee86084b6b98f6f62372eed418f65597 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 7 Sep 2022 10:24:07 -0500 Subject: [PATCH 204/551] Better logging for failure to get connection from pool (#161) --- src/client.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index 7fe633c08..a1c4600cc 100644 --- a/src/client.rs +++ b/src/client.rs @@ -748,7 +748,9 @@ where } error_response(&mut self.write, "could not get connection from the pool") .await?; - error!("Could not get connection from pool: {:?}", err); + + error!("Could not get connection from pool: {{ pool_name: {:?}, username: {:?}, shard: {:?}, role: \"{:?}\", error: \"{:?}\" }}", + self.pool_name.clone(), self.username.clone(), query_router.shard(), query_router.role(), err); continue; } }; From 68dffc1f2e2678e86666e207f33d354468ff25df Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 7 Sep 2022 09:22:52 -0700 Subject: [PATCH 205/551] Send signal even if process is gone (#162) * Send signal even if process is gone * hmm * hmm --- tests/python/tests.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/python/tests.py b/tests/python/tests.py index 092fc8cc9..7c9907988 100644 --- a/tests/python/tests.py +++ b/tests/python/tests.py @@ -18,9 +18,14 @@ def pgcat_start(): def pg_cat_send_signal(signal: signal.Signals): - for proc in psutil.process_iter(["pid", "name"]): - if "pgcat" == proc.name(): - os.kill(proc.pid, signal) + try: + for proc in psutil.process_iter(["pid", "name"]): + if "pgcat" == proc.name(): + os.kill(proc.pid, signal) + except Exception as e: + # The process can be gone when we send this signal + print(e) + if signal == signal.SIGTERM: # Returns 0 if pgcat process exists time.sleep(2) From 0cdbe0da83c0f4592ba90e827fae20120523c032 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 7 Sep 2022 22:37:17 -0500 Subject: [PATCH 206/551] Clean connection state up after protocol named prepared statement (#163) * Clean connection state up after protocol named prepared statement * Avoid cloning + add test * fmt --- src/client.rs | 17 +++++++++++++++++ src/server.rs | 5 +++++ tests/ruby/misc_spec.rb | 7 +++++++ 3 files changed, 29 insertions(+) diff --git a/src/client.rs b/src/client.rs index a1c4600cc..3aac72c7b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -879,6 +879,23 @@ where self.buffer.put(&original[..]); + // Clone after freeze does not allocate + let first_message_code = (*self.buffer.get(0).unwrap_or(&0)) as char; + + // Almost certainly true + if first_message_code == 'P' { + // Message layout + // P followed by 32 int followed by null-terminated statement name + // So message code should be in offset 0 of the buffer, first character + // in prepared statement name would be index 5 + let first_char_in_name = *self.buffer.get(5).unwrap_or(&0); + if first_char_in_name != 0 { + // This is a named prepared statement + // Server connection state will need to be cleared at checkin + server.mark_dirty(); + } + } + self.send_and_receive_loop( code, self.buffer.clone(), diff --git a/src/server.rs b/src/server.rs index e04676ec7..af3d680e7 100644 --- a/src/server.rs +++ b/src/server.rs @@ -638,6 +638,11 @@ impl Server { pub fn last_activity(&self) -> SystemTime { self.last_activity } + + // Marks a connection as needing DISCARD ALL at checkin + pub fn mark_dirty(&mut self) { + self.needs_cleanup = true; + } } impl Drop for Server { diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index d5b529a9e..6e79e1a4f 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -159,6 +159,12 @@ conn.async_exec("PREPARE prepared_q (int) AS SELECT $1") conn.close end + + 15.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.prepare("prepared_q", "SELECT $1") + conn.close + end end it "Does not send DISCARD ALL unless necessary" do @@ -166,6 +172,7 @@ conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) conn.async_exec("SET SERVER ROLE to 'primary'") conn.async_exec("SELECT 1") + conn.exec_params("SELECT $1", [1]) conn.close end From b8e09188aec580cb6df237aaa6f7ade556f9be95 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 8 Sep 2022 08:12:37 -0700 Subject: [PATCH 207/551] Add Discord link (#164) * Add Discord link * move it up * :) * hmm * hmm * :O --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8d4c77fa1..dfecc0abb 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,11 @@ -# PgCat +![PgCat](./pgcat3.png) -[![CircleCI](https://circleci.com/gh/levkk/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/levkk/pgcat/tree/main) +##### PgCat: PostgreSQL at petabyte scale -![PgCat](./pgcat3.png) +[![CircleCI](https://circleci.com/gh/levkk/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/levkk/pgcat/tree/main) + + Join our Discord! + PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover support. From 643395b32caa7fce258f878d217fcc4a6f4184bb Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 14 Sep 2022 10:20:41 -0500 Subject: [PATCH 208/551] Add SHOW CLIENTS / SHOW SERVERS + Stats refactor and tests (#159) * wip * Main Thread Panic when swarmed with clients * fix * fix * 1024 * fix * remove test * Add SHOW CLIENTS * revert * fmt * Refactor + tests * fmt * add test * Add SHOW SERVERS + Make PR unreviewable * prometheus * add state to clients and servers * fmt * Add application_name to server stats * Add tests for waiting clients * Docs * remove comment * comments * typo * cleanup * CI --- src/admin.rs | 197 ++++-- src/client.rs | 40 +- src/pool.rs | 55 +- src/prometheus.rs | 4 +- src/server.rs | 20 +- src/stats.rs | 999 ++++++++++++++++++++-------- tests/ruby/admin_spec.rb | 214 ++++++ tests/ruby/helpers/pgcat_helper.rb | 44 ++ tests/ruby/helpers/pgcat_process.rb | 2 +- 9 files changed, 1203 insertions(+), 372 deletions(-) create mode 100644 tests/ruby/admin_spec.rb diff --git a/src/admin.rs b/src/admin.rs index 4576d1681..ed2d3de3a 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -2,12 +2,15 @@ use bytes::{Buf, BufMut, BytesMut}; use log::{info, trace}; use std::collections::HashMap; +use tokio::time::Instant; use crate::config::{get_config, reload_config, VERSION}; use crate::errors::Error; use crate::messages::*; use crate::pool::get_all_pools; -use crate::stats::get_stats; +use crate::stats::{ + get_address_stats, get_client_stats, get_pool_stats, get_server_stats, ClientState, ServerState, +}; use crate::ClientServerMap; pub fn generate_server_info_for_admin() -> BytesMut { @@ -72,6 +75,14 @@ where trace!("SHOW POOLS"); show_pools(stream).await } + "CLIENTS" => { + trace!("SHOW CLIENTS"); + show_clients(stream).await + } + "SERVERS" => { + trace!("SHOW SERVERS"); + show_servers(stream).await + } "STATS" => { trace!("SHOW STATS"); show_stats(stream).await @@ -91,7 +102,8 @@ async fn show_lists(stream: &mut T) -> Result<(), Error> where T: tokio::io::AsyncWrite + std::marker::Unpin, { - let stats = get_stats(); + let client_stats = get_client_stats(); + let server_stats = get_server_stats(); let columns = vec![("list", DataType::Text), ("items", DataType::Int4)]; @@ -111,18 +123,18 @@ where res.put(data_row(&vec!["pools".to_string(), databases.to_string()])); res.put(data_row(&vec![ "free_clients".to_string(), - stats + client_stats .keys() - .map(|address_id| stats[&address_id]["cl_idle"]) - .sum::() + .filter(|client_id| client_stats.get(client_id).unwrap().state == ClientState::Idle) + .count() .to_string(), ])); res.put(data_row(&vec![ "used_clients".to_string(), - stats + client_stats .keys() - .map(|address_id| stats[&address_id]["cl_active"]) - .sum::() + .filter(|client_id| client_stats.get(client_id).unwrap().state == ClientState::Active) + .count() .to_string(), ])); res.put(data_row(&vec![ @@ -131,18 +143,18 @@ where ])); res.put(data_row(&vec![ "free_servers".to_string(), - stats + server_stats .keys() - .map(|address_id| stats[&address_id]["sv_idle"]) - .sum::() + .filter(|server_id| server_stats.get(server_id).unwrap().state == ServerState::Idle) + .count() .to_string(), ])); res.put(data_row(&vec![ "used_servers".to_string(), - stats + server_stats .keys() - .map(|address_id| stats[&address_id]["sv_active"]) - .sum::() + .filter(|server_id| server_stats.get(server_id).unwrap().state == ServerState::Active) + .count() .to_string(), ])); res.put(data_row(&vec!["dns_names".to_string(), "0".to_string()])); @@ -182,11 +194,12 @@ async fn show_pools(stream: &mut T) -> Result<(), Error> where T: tokio::io::AsyncWrite + std::marker::Unpin, { - let stats = get_stats(); + let all_pool_stats = get_pool_stats(); let columns = vec![ ("database", DataType::Text), ("user", DataType::Text), + ("pool_mode", DataType::Text), ("cl_idle", DataType::Numeric), ("cl_active", DataType::Numeric), ("cl_waiting", DataType::Numeric), @@ -198,32 +211,27 @@ where ("sv_login", DataType::Numeric), ("maxwait", DataType::Numeric), ("maxwait_us", DataType::Numeric), - ("pool_mode", DataType::Text), ]; let mut res = BytesMut::new(); res.put(row_description(&columns)); - for (_, pool) in get_all_pools() { - let pool_config = &pool.settings; - for shard in 0..pool.shards() { - for server in 0..pool.servers(shard) { - let address = pool.address(shard, server); - let stats = match stats.get(&address.id) { - Some(stats) => stats.clone(), - None => HashMap::new(), - }; + for ((pool_name, username), pool) in get_all_pools() { + let def = HashMap::default(); + let pool_stats = all_pool_stats + .get(&(pool_name.clone(), username.clone())) + .unwrap_or(&def); - let mut row = vec![address.name(), pool_config.user.username.clone()]; - - for column in &columns[2..columns.len() - 1] { - let value = stats.get(column.0).unwrap_or(&0).to_string(); - row.push(value); - } - - row.push(pool_config.pool_mode.to_string()); - res.put(data_row(&row)); - } + let pool_config = &pool.settings; + let mut row = vec![ + pool_name.clone(), + username.clone(), + pool_config.pool_mode.to_string(), + ]; + for column in &columns[3..columns.len()] { + let value = pool_stats.get(column.0).unwrap_or(&0).to_string(); + row.push(value); } + res.put(data_row(&row)); } res.put(command_complete("SHOW")); @@ -387,6 +395,7 @@ where T: tokio::io::AsyncWrite + std::marker::Unpin, { let columns = vec![ + ("instance", DataType::Text), ("database", DataType::Text), ("user", DataType::Text), ("total_xact_count", DataType::Numeric), @@ -396,32 +405,32 @@ where ("total_xact_time", DataType::Numeric), ("total_query_time", DataType::Numeric), ("total_wait_time", DataType::Numeric), + ("total_errors", DataType::Numeric), ("avg_xact_count", DataType::Numeric), ("avg_query_count", DataType::Numeric), ("avg_recv", DataType::Numeric), ("avg_sent", DataType::Numeric), + ("avg_errors", DataType::Numeric), ("avg_xact_time", DataType::Numeric), ("avg_query_time", DataType::Numeric), ("avg_wait_time", DataType::Numeric), ]; - let stats = get_stats(); + let all_stats = get_address_stats(); let mut res = BytesMut::new(); res.put(row_description(&columns)); - for ((_db_name, username), pool) in get_all_pools() { + for ((db, username), pool) in get_all_pools() { for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { let address = pool.address(shard, server); - let stats = match stats.get(&address.id) { + let stats = match all_stats.get(&address.id) { Some(stats) => stats.clone(), None => HashMap::new(), }; - let mut row = vec![address.name()]; - row.push(username.clone()); - - for column in &columns[2..] { + let mut row = vec![address.name(), db.clone(), username.clone()]; + for column in &columns[3..] { row.push(stats.get(column.0).unwrap_or(&0).to_string()); } @@ -439,3 +448,107 @@ where write_all_half(stream, res).await } + +/// Show currently connected clients +async fn show_clients(stream: &mut T) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let columns = vec![ + ("client_id", DataType::Text), + ("database", DataType::Text), + ("user", DataType::Text), + ("application_name", DataType::Text), + ("state", DataType::Text), + ("transaction_count", DataType::Numeric), + ("query_count", DataType::Numeric), + ("error_count", DataType::Numeric), + ("age_seconds", DataType::Numeric), + ]; + + let new_map = get_client_stats(); + let mut res = BytesMut::new(); + res.put(row_description(&columns)); + + for (_, client) in new_map { + let row = vec![ + format!("{:#010X}", client.client_id), + client.pool_name, + client.username, + client.application_name.clone(), + client.state.to_string(), + client.transaction_count.to_string(), + client.query_count.to_string(), + client.error_count.to_string(), + Instant::now() + .duration_since(client.connect_time) + .as_secs() + .to_string(), + ]; + + res.put(data_row(&row)); + } + + res.put(command_complete("SHOW")); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} + +/// Show currently connected servers +async fn show_servers(stream: &mut T) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let columns = vec![ + ("server_id", DataType::Text), + ("database_name", DataType::Text), + ("user", DataType::Text), + ("address_id", DataType::Text), + ("application_name", DataType::Text), + ("state", DataType::Text), + ("transaction_count", DataType::Numeric), + ("query_count", DataType::Numeric), + ("bytes_sent", DataType::Numeric), + ("bytes_received", DataType::Numeric), + ("age_seconds", DataType::Numeric), + ]; + + let new_map = get_server_stats(); + let mut res = BytesMut::new(); + res.put(row_description(&columns)); + + for (_, server) in new_map { + let row = vec![ + format!("{:#010X}", server.server_id), + server.pool_name, + server.username, + server.address_name, + server.application_name, + server.state.to_string(), + server.transaction_count.to_string(), + server.query_count.to_string(), + server.bytes_sent.to_string(), + server.bytes_received.to_string(), + Instant::now() + .duration_since(server.connect_time) + .as_secs() + .to_string(), + ]; + + res.put(data_row(&row)); + } + + res.put(command_complete("SHOW")); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, res).await +} diff --git a/src/client.rs b/src/client.rs index 3aac72c7b..c07a21f06 100644 --- a/src/client.rs +++ b/src/client.rs @@ -577,6 +577,12 @@ where // The query router determines where the query is going to go, // e.g. primary, replica, which shard. let mut query_router = QueryRouter::new(); + self.stats.client_register( + self.process_id, + self.pool_name.clone(), + self.username.clone(), + self.application_name.clone(), + ); // Our custom protocol loop. // We expect the client to either start a transaction with regular queries @@ -764,15 +770,12 @@ where server.claim(self.process_id, self.secret_key); self.connected_to_server = true; - // Update statistics. - if let Some(last_address_id) = self.last_address_id { - self.stats - .client_disconnecting(self.process_id, last_address_id); - } - self.stats.client_active(self.process_id, address.id); + // Update statistics + self.stats + .client_active(self.process_id, server.server_id()); self.last_address_id = Some(address.id); - self.last_server_id = Some(server.process_id()); + self.last_server_id = Some(server.server_id()); debug!( "Client {:?} talking to server {:?}", @@ -830,7 +833,7 @@ where if !server.in_transaction() { // Report transaction executed statistics. - self.stats.transaction(self.process_id, address.id); + self.stats.transaction(self.process_id, server.server_id()); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -908,7 +911,7 @@ where self.buffer.clear(); if !server.in_transaction() { - self.stats.transaction(self.process_id, address.id); + self.stats.transaction(self.process_id, server.server_id()); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -943,7 +946,7 @@ where }; if !server.in_transaction() { - self.stats.transaction(self.process_id, address.id); + self.stats.transaction(self.process_id, server.server_id()); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -964,11 +967,11 @@ where // The server is no longer bound to us, we can't cancel it's queries anymore. debug!("Releasing server back into the pool"); server.checkin_cleanup().await?; - self.stats.server_idle(server.process_id(), address.id); + self.stats.server_idle(server.server_id()); self.connected_to_server = false; self.release(); - self.stats.client_idle(self.process_id, address.id); + self.stats.client_idle(self.process_id); } } @@ -1010,7 +1013,7 @@ where } // Report query executed statistics. - self.stats.query(self.process_id, address.id); + self.stats.query(self.process_id, server.server_id()); Ok(()) } @@ -1091,14 +1094,9 @@ impl Drop for Client { // Dirty shutdown // TODO: refactor, this is not the best way to handle state management. - if let Some(address_id) = self.last_address_id { - self.stats.client_disconnecting(self.process_id, address_id); - - if self.connected_to_server { - if let Some(process_id) = self.last_server_id { - self.stats.server_idle(process_id, address_id); - } - } + self.stats.client_disconnecting(self.process_id); + if self.connected_to_server && self.last_server_id.is_some() { + self.stats.server_idle(self.last_server_id.unwrap()); } } } diff --git a/src/pool.rs b/src/pool.rs index dea29ad24..34af354a4 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -332,7 +332,7 @@ impl ConnectionPool { } // Indicate we're waiting on a server connection from a pool. - self.stats.client_waiting(process_id, address.id); + self.stats.client_waiting(process_id); // Check if we can connect let mut conn = match self.databases[address.shard][address.address_index] @@ -343,8 +343,7 @@ impl ConnectionPool { Err(err) => { error!("Banning instance {:?}, error: {:?}", address, err); self.ban(&address, process_id); - self.stats - .checkout_time(now.elapsed().as_micros(), process_id, address.id); + self.stats.client_checkout_error(process_id, address.id); continue; } }; @@ -361,14 +360,14 @@ impl ConnectionPool { // Health checks are pretty expensive. if !require_healthcheck { self.stats - .checkout_time(now.elapsed().as_micros(), process_id, address.id); - self.stats.server_active(conn.process_id(), address.id); + .checkout_time(now.elapsed().as_micros(), process_id, server.server_id()); + self.stats.server_active(process_id, server.server_id()); return Ok((conn, address.clone())); } debug!("Running health check on server {:?}", address); - self.stats.server_tested(server.process_id(), address.id); + self.stats.server_tested(server.server_id()); match tokio::time::timeout( tokio::time::Duration::from_millis(healthcheck_timeout), @@ -379,9 +378,12 @@ impl ConnectionPool { // Check if health check succeeded. Ok(res) => match res { Ok(_) => { - self.stats - .checkout_time(now.elapsed().as_micros(), process_id, address.id); - self.stats.server_active(conn.process_id(), address.id); + self.stats.checkout_time( + now.elapsed().as_micros(), + process_id, + conn.server_id(), + ); + self.stats.server_active(process_id, conn.server_id()); return Ok((conn, address.clone())); } @@ -421,10 +423,9 @@ impl ConnectionPool { /// Ban an address (i.e. replica). It no longer will serve /// traffic for any new transactions. Existing transactions on that replica /// will finish successfully or error out to the clients. - pub fn ban(&self, address: &Address, process_id: i32) { - self.stats.client_disconnecting(process_id, address.id); - + pub fn ban(&self, address: &Address, client_id: i32) { error!("Banning {:?}", address); + self.stats.client_ban_error(client_id, address.id); let now = chrono::offset::Utc::now().naive_utc(); let mut guard = self.banlist.write(); @@ -560,14 +561,20 @@ impl ManageConnection for ServerPool { /// Attempts to create a new connection. async fn connect(&self) -> Result { info!("Creating a new server connection {:?}", self.address); + let server_id = rand::random::(); - // Put a temporary process_id into the stats - // for server login. - let process_id = rand::random::(); - self.stats.server_login(process_id, self.address.id); + self.stats.server_register( + server_id, + self.address.id, + self.address.name(), + self.address.pool_name.clone(), + self.address.username.clone(), + ); + self.stats.server_login(server_id); // Connect to the PostgreSQL server. match Server::startup( + server_id, &self.address, &self.user, &self.database, @@ -577,13 +584,11 @@ impl ManageConnection for ServerPool { .await { Ok(conn) => { - // Remove the temporary process_id from the stats. - self.stats.server_disconnecting(process_id, self.address.id); + self.stats.server_idle(server_id); Ok(conn) } Err(err) => { - // Remove the temporary process_id from the stats. - self.stats.server_disconnecting(process_id, self.address.id); + self.stats.server_disconnecting(server_id); Err(err) } } @@ -608,6 +613,11 @@ pub fn get_pool(db: String, user: String) -> Option { } } +/// Get a pointer to all configured pools. +pub fn get_all_pools() -> HashMap<(String, String), ConnectionPool> { + return (*(*POOLS.load())).clone(); +} + /// How many total servers we have in the config. pub fn get_number_of_addresses() -> usize { get_all_pools() @@ -615,8 +625,3 @@ pub fn get_number_of_addresses() -> usize { .map(|(_, pool)| pool.databases()) .sum() } - -/// Get a pointer to all configured pools. -pub fn get_all_pools() -> HashMap<(String, String), ConnectionPool> { - return (*(*POOLS.load())).clone(); -} diff --git a/src/prometheus.rs b/src/prometheus.rs index 4e3dc3aaa..d54ba3291 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -8,7 +8,7 @@ use std::net::SocketAddr; use crate::config::Address; use crate::pool::get_all_pools; -use crate::stats::get_stats; +use crate::stats::get_address_stats; struct MetricHelpType { help: &'static str, @@ -164,7 +164,7 @@ impl PrometheusMetric { async fn prometheus_stats(request: Request) -> Result, hyper::http::Error> { match (request.method(), request.uri().path()) { (&Method::GET, "/metrics") => { - let stats = get_stats(); + let stats: HashMap> = get_address_stats(); let mut lines = Vec::new(); for (_, pool) in get_all_pools() { diff --git a/src/server.rs b/src/server.rs index af3d680e7..d8d23d6c7 100644 --- a/src/server.rs +++ b/src/server.rs @@ -20,6 +20,8 @@ use crate::ClientServerMap; /// Server state. pub struct Server { + server_id: i32, + /// Server host, e.g. localhost, /// port, e.g. 5432, and role, e.g. primary or replica. address: Address, @@ -72,6 +74,7 @@ impl Server { /// Pretend to be the Postgres client and connect to the server given host, port and credentials. /// Perform the authentication and return the server in a ready for query state. pub async fn startup( + server_id: i32, address: &Address, user: &User, database: &str, @@ -315,6 +318,7 @@ impl Server { write: write, buffer: BytesMut::with_capacity(8196), server_info: server_info, + server_id: server_id, process_id: process_id, secret_key: secret_key, in_transaction: false, @@ -372,8 +376,7 @@ impl Server { /// Send messages to the server from the client. pub async fn send(&mut self, messages: BytesMut) -> Result<(), Error> { - self.stats - .data_sent(messages.len(), self.process_id, self.address.id); + self.stats.data_sent(messages.len(), self.server_id); match write_all_half(&mut self.write, messages).await { Ok(_) => { @@ -505,8 +508,7 @@ impl Server { let bytes = self.buffer.clone(); // Keep track of how much data we got from the server for stats. - self.stats - .data_received(bytes.len(), self.process_id, self.address.id); + self.stats.data_received(bytes.len(), self.server_id); // Clear the buffer for next query. self.buffer.clear(); @@ -629,9 +631,10 @@ impl Server { self.address.clone() } - /// Get the server's unique identifier. - pub fn process_id(&self) -> i32 { - self.process_id + /// Get the server connection identifier + /// Used to uniquely identify connection in statistics + pub fn server_id(&self) -> i32 { + self.server_id } // Get server's latest response timestamp @@ -650,8 +653,7 @@ impl Drop for Server { /// the socket is in non-blocking mode, so it may not be ready /// for a write. fn drop(&mut self) { - self.stats - .server_disconnecting(self.process_id(), self.address.id); + self.stats.server_disconnecting(self.server_id); let mut bytes = BytesMut::with_capacity(4); bytes.put_u8(b'X'); diff --git a/src/stats.rs b/src/stats.rs index fde4071b8..9ab71677c 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,45 +1,219 @@ use arc_swap::ArcSwap; /// Statistics and reporting. -use log::{error, info, trace}; +use log::{error, info, trace, warn}; use once_cell::sync::Lazy; -use parking_lot::Mutex; use std::collections::HashMap; +use std::sync::Arc; use tokio::sync::mpsc::error::TrySendError; use tokio::sync::mpsc::{channel, Receiver, Sender}; - -use crate::pool::get_number_of_addresses; - +use tokio::time::Instant; + +use crate::pool::{get_all_pools, get_number_of_addresses}; +use crate::server; + +/// Convenience types for various stats +type ClientStatesLookup = HashMap; +type ServerStatesLookup = HashMap; +type PoolStatsLookup = HashMap<(String, String), HashMap>; +type AddressStatsLookup = HashMap>; + +/// Stats for individual client connections updated every second +/// Used in SHOW CLIENTS. +static LATEST_CLIENT_STATS: Lazy> = + Lazy::new(|| ArcSwap::from_pointee(ClientStatesLookup::default())); + +/// Stats for individual server connections updated every second +/// Used in SHOW SERVERS. +static LATEST_SERVER_STATS: Lazy> = + Lazy::new(|| ArcSwap::from_pointee(ServerStatesLookup::default())); + +/// Aggregate stats for each pool (a pool is identified by database name and username) updated every second +/// Used in SHOW POOLS. +static LATEST_POOL_STATS: Lazy> = + Lazy::new(|| ArcSwap::from_pointee(PoolStatsLookup::default())); + +/// Aggregate stats for individual database instances, updated every second, averages are calculated every 15 +/// Used in SHOW STATS. +static LATEST_ADDRESS_STATS: Lazy> = + Lazy::new(|| ArcSwap::from_pointee(AddressStatsLookup::default())); + +/// The statistics reporter. An instance is given to each possible source of statistics, +/// e.g. clients, servers, connection pool. pub static REPORTER: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Reporter::default())); -/// Latest stats updated every second; used in SHOW STATS and other admin commands. -static LATEST_STATS: Lazy>>> = - Lazy::new(|| Mutex::new(HashMap::new())); - /// Statistics period used for average calculations. /// 15 seconds. static STAT_PERIOD: u64 = 15000; +/// The various states that a client can be in +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ClientState { + Idle, + Waiting, + Active, +} +impl std::fmt::Display for ClientState { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match *self { + ClientState::Idle => write!(f, "idle"), + ClientState::Waiting => write!(f, "waiting"), + ClientState::Active => write!(f, "active"), + } + } +} + +/// The various states that a server can be in +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ServerState { + Login, + Active, + Tested, + Idle, +} +impl std::fmt::Display for ServerState { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match *self { + ServerState::Login => write!(f, "login"), + ServerState::Active => write!(f, "active"), + ServerState::Tested => write!(f, "tested"), + ServerState::Idle => write!(f, "idle"), + } + } +} + +/// Information we keep track off which can be queried by SHOW CLIENTS +#[derive(Debug, Clone)] +pub struct ClientInformation { + pub state: ClientState, + pub connect_time: Instant, + + /// A random integer assigned to the client and used by stats to track the client + pub client_id: i32, + + pub application_name: String, + pub username: String, + pub pool_name: String, + + /// Total time spent waiting for a connection from pool, measures in microseconds + pub total_wait_time: u64, + + pub transaction_count: u64, + pub query_count: u64, + pub error_count: u64, +} + +/// Information we keep track off which can be queried by SHOW SERVERS +#[derive(Debug, Clone)] +pub struct ServerInformation { + pub state: ServerState, + pub connect_time: Instant, + + /// A random integer assigned to the server and used by stats to track the server + pub server_id: i32, + + pub address_name: String, + pub address_id: usize, + + pub username: String, + pub pool_name: String, + pub application_name: String, + + pub bytes_sent: u64, + pub bytes_received: u64, + + pub transaction_count: u64, + pub query_count: u64, + pub error_count: u64, +} + /// The names for the events reported /// to the statistics collector. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] enum EventName { - CheckoutTime, - Query, - Transaction, - DataSent, - DataReceived, - ClientWaiting, - ClientActive, - ClientIdle, - ClientDisconnecting, - ServerActive, - ServerIdle, - ServerTested, - ServerLogin, - ServerDisconnecting, - UpdateStats, - UpdateAverages, + CheckoutTime { + client_id: i32, + server_id: i32, + }, + Query { + client_id: i32, + server_id: i32, + }, + Transaction { + client_id: i32, + server_id: i32, + }, + + DataSentToServer { + server_id: i32, + }, + DataReceivedFromServer { + server_id: i32, + }, + + ClientRegistered { + client_id: i32, + pool_name: String, + username: String, + application_name: String, + }, + ClientIdle { + client_id: i32, + }, + ClientWaiting { + client_id: i32, + }, + ClientActive { + client_id: i32, + #[allow(dead_code)] + server_id: i32, + }, + ClientDisconnecting { + client_id: i32, + }, + ClientCheckoutError { + client_id: i32, + #[allow(dead_code)] + address_id: usize, + }, + ClientBanError { + client_id: i32, + #[allow(dead_code)] + address_id: usize, + }, + + ServerRegistered { + server_id: i32, + address_id: usize, + address_name: String, + pool_name: String, + username: String, + }, + ServerLogin { + server_id: i32, + }, + ServerIdle { + server_id: i32, + }, + ServerTested { + server_id: i32, + }, + ServerActive { + #[allow(dead_code)] + client_id: i32, + server_id: i32, + }, + ServerDisconnecting { + server_id: i32, + }, + + UpdateStats { + pool_name: String, + username: String, + }, + UpdateAverages { + address_id: usize, + }, } /// Event data sent to the collector @@ -51,12 +225,6 @@ pub struct Event { /// The value being reported. Meaning differs based on event name. value: i64, - - /// The client or server connection reporting the event. - process_id: i32, - - /// The server the client is connected to. - address_id: usize, } /// The statistics reporter. An instance is given @@ -82,14 +250,15 @@ impl Reporter { /// Send statistics to the task keeping track of stats. fn send(&self, event: Event) { - let name = event.name; - let result = self.tx.try_send(event); + let name = event.name.clone(); + let result = self.tx.try_send(event.clone()); match result { Ok(_) => trace!( - "{:?} event reported successfully, capacity: {}", + "{:?} event reported successfully, capacity: {} {:?}", name, - self.tx.capacity() + self.tx.capacity(), + event ), Err(err) => match err { @@ -99,190 +268,218 @@ impl Reporter { }; } - /// Report a query executed by a client against - /// a server identified by the `address_id`. - pub fn query(&self, process_id: i32, address_id: usize) { + /// Report a query executed by a client against a server + pub fn query(&self, client_id: i32, server_id: i32) { let event = Event { - name: EventName::Query, + name: EventName::Query { + client_id, + server_id, + }, value: 1, - process_id: process_id, - address_id: address_id, }; - self.send(event); } - /// Report a transaction executed by a client against - /// a server identified by the `address_id`. - pub fn transaction(&self, process_id: i32, address_id: usize) { + /// Report a transaction executed by a client a server + /// we report each individual queries outside a transaction as a transaction + /// We only count the initial BEGIN as a transaction, all queries within do not + /// count as transactions + pub fn transaction(&self, client_id: i32, server_id: i32) { let event = Event { - name: EventName::Transaction, + name: EventName::Transaction { + client_id, + server_id, + }, value: 1, - process_id: process_id, - address_id: address_id, }; + self.send(event); + } + /// Report data sent to a server + pub fn data_sent(&self, amount_bytes: usize, server_id: i32) { + let event = Event { + name: EventName::DataSentToServer { server_id }, + value: amount_bytes as i64, + }; self.send(event) } - /// Report data sent to a server identified by `address_id`. - /// The `amount` is measured in bytes. - pub fn data_sent(&self, amount: usize, process_id: i32, address_id: usize) { + /// Report data received from a server + pub fn data_received(&self, amount_bytes: usize, server_id: i32) { let event = Event { - name: EventName::DataSent, - value: amount as i64, - process_id: process_id, - address_id: address_id, + name: EventName::DataReceivedFromServer { server_id }, + value: amount_bytes as i64, }; - self.send(event) } - /// Report data received from a server identified by `address_id`. - /// The `amount` is measured in bytes. - pub fn data_received(&self, amount: usize, process_id: i32, address_id: usize) { + /// Reportes the time spent by a client waiting to get a healthy connection from the pool + pub fn checkout_time(&self, microseconds: u128, client_id: i32, server_id: i32) { let event = Event { - name: EventName::DataReceived, - value: amount as i64, - process_id: process_id, - address_id: address_id, + name: EventName::CheckoutTime { + client_id, + server_id, + }, + value: microseconds as i64, }; - self.send(event) } - /// Time spent waiting to get a healthy connection from the pool - /// for a server identified by `address_id`. - /// Measured in milliseconds. - pub fn checkout_time(&self, ms: u128, process_id: i32, address_id: usize) { + /// Register a client with the stats system. The stats system uses client_id + /// to track and aggregate statistics from all source that relate to that client + pub fn client_register( + &self, + client_id: i32, + pool_name: String, + username: String, + app_name: String, + ) { let event = Event { - name: EventName::CheckoutTime, - value: ms as i64, - process_id: process_id, - address_id: address_id, + name: EventName::ClientRegistered { + client_id, + pool_name: pool_name.clone(), + username: username.clone(), + application_name: app_name.clone(), + }, + value: 1, }; + self.send(event); + } + /// Reports a client is waiting for a connection + pub fn client_waiting(&self, client_id: i32) { + let event = Event { + name: EventName::ClientWaiting { client_id }, + value: 1, + }; self.send(event) } - /// Reports a client identified by `process_id` waiting for a connection - /// to a server identified by `address_id`. - pub fn client_waiting(&self, process_id: i32, address_id: usize) { + /// Reports a client has had the server assigned to it be banned + pub fn client_ban_error(&self, client_id: i32, address_id: usize) { let event = Event { - name: EventName::ClientWaiting, + name: EventName::ClientBanError { + client_id, + address_id, + }, value: 1, - process_id: process_id, - address_id: address_id, }; - self.send(event) } - /// Reports a client identified by `process_id` is done waiting for a connection - /// to a server identified by `address_id` and is about to query the server. - pub fn client_active(&self, process_id: i32, address_id: usize) { + /// Reports a client has failed to obtain a connection from a connection pool + pub fn client_checkout_error(&self, client_id: i32, address_id: usize) { let event = Event { - name: EventName::ClientActive, + name: EventName::ClientCheckoutError { + client_id, + address_id, + }, value: 1, - process_id: process_id, - address_id: address_id, }; - self.send(event) } - /// Reports a client identified by `process_id` is done querying the server - /// identified by `address_id` and is no longer active. - pub fn client_idle(&self, process_id: i32, address_id: usize) { + /// Reports a client is done waiting for a connection and is about to query the server. + pub fn client_active(&self, client_id: i32, server_id: i32) { let event = Event { - name: EventName::ClientIdle, + name: EventName::ClientActive { + client_id, + server_id, + }, value: 1, - process_id: process_id, - address_id: address_id, }; - self.send(event) } - /// Reports a client identified by `process_id` is disconecting from the pooler. - /// The last server it was connected to is identified by `address_id`. - pub fn client_disconnecting(&self, process_id: i32, address_id: usize) { + /// Reports a client is done querying the server and is no longer assigned a server connection + pub fn client_idle(&self, client_id: i32) { let event = Event { - name: EventName::ClientDisconnecting, + name: EventName::ClientIdle { client_id }, value: 1, - process_id: process_id, - address_id: address_id, }; - self.send(event) } - /// Reports a server connection identified by `process_id` for - /// a configured server identified by `address_id` is actively used - /// by a client. - pub fn server_active(&self, process_id: i32, address_id: usize) { + /// Reports a client is disconecting from the pooler. + pub fn client_disconnecting(&self, client_id: i32) { let event = Event { - name: EventName::ServerActive, + name: EventName::ClientDisconnecting { client_id }, value: 1, - process_id: process_id, - address_id: address_id, }; - self.send(event) } - /// Reports a server connection identified by `process_id` for - /// a configured server identified by `address_id` is no longer - /// actively used by a client and is now idle. - pub fn server_idle(&self, process_id: i32, address_id: usize) { + /// Register a server connection with the stats system. The stats system uses server_id + /// to track and aggregate statistics from all source that relate to that server + pub fn server_register( + &self, + server_id: i32, + address_id: usize, + address_name: String, + pool_name: String, + username: String, + ) { let event = Event { - name: EventName::ServerIdle, + name: EventName::ServerRegistered { + server_id, + address_id, + address_name, + pool_name, + username, + }, value: 1, - process_id: process_id, - address_id: address_id, }; + self.send(event); + } + /// Reports a server connection has been assigned to a client that + /// is about to query the server + pub fn server_active(&self, client_id: i32, server_id: i32) { + let event = Event { + name: EventName::ServerActive { + client_id, + server_id, + }, + value: 1, + }; self.send(event) } - /// Reports a server connection identified by `process_id` for - /// a configured server identified by `address_id` is attempting - /// to login. - pub fn server_login(&self, process_id: i32, address_id: usize) { + /// Reports a server connection is no longer assigned to a client + /// and is available for the next client to pick it up + pub fn server_idle(&self, server_id: i32) { let event = Event { - name: EventName::ServerLogin, + name: EventName::ServerIdle { server_id }, value: 1, - process_id: process_id, - address_id: address_id, }; + self.send(event) + } + /// Reports a server connection is attempting to login. + pub fn server_login(&self, server_id: i32) { + let event = Event { + name: EventName::ServerLogin { server_id }, + value: 1, + }; self.send(event) } - /// Reports a server connection identified by `process_id` for - /// a configured server identified by `address_id` is being - /// tested before being given to a client. - pub fn server_tested(&self, process_id: i32, address_id: usize) { + /// Reports a server connection is being tested before being given to a client. + pub fn server_tested(&self, server_id: i32) { let event = Event { - name: EventName::ServerTested, + name: EventName::ServerTested { server_id }, value: 1, - process_id: process_id, - address_id: address_id, }; self.send(event) } - /// Reports a server connection identified by `process_id` is disconecting from the pooler. - /// The configured server it was connected to is identified by `address_id`. - pub fn server_disconnecting(&self, process_id: i32, address_id: usize) { + /// Reports a server connection is disconecting from the pooler. + pub fn server_disconnecting(&self, server_id: i32) { let event = Event { - name: EventName::ServerDisconnecting, + name: EventName::ServerDisconnecting { server_id }, value: 1, - process_id: process_id, - address_id: address_id, }; - self.send(event) } } @@ -310,55 +507,26 @@ impl Collector { pub async fn collect(&mut self) { info!("Events reporter started"); - let stats_template = HashMap::from([ - ("total_query_count", 0), - ("total_query_time", 0), - ("total_received", 0), - ("total_sent", 0), - ("total_xact_count", 0), - ("total_xact_time", 0), - ("total_wait_time", 0), - ("avg_query_count", 0), - ("avg_query_time", 0), - ("avg_recv", 0), - ("avg_sent", 0), - ("avg_xact_count", 0), - ("avg_xact_time", 0), - ("avg_wait_time", 0), - ("maxwait_us", 0), - ("maxwait", 0), - ("cl_waiting", 0), - ("cl_active", 0), - ("cl_idle", 0), - ("sv_idle", 0), - ("sv_active", 0), - ("sv_login", 0), - ("sv_tested", 0), - ]); - - let mut stats = HashMap::new(); - - // Stats saved after each iteration of the flush event. Used in calculation - // of averages in the last flush period. - let mut old_stats: HashMap> = HashMap::new(); - - // Track which state the client and server are at any given time. - let mut client_server_states: HashMap> = HashMap::new(); - - // Flush stats to StatsD and calculate averages every 15 seconds. + let mut client_states = ClientStatesLookup::default(); + let mut server_states = ServerStatesLookup::default(); + let mut pool_stat_lookup = PoolStatsLookup::default(); + + let mut address_stat_lookup = AddressStatsLookup::default(); + let mut address_old_stat_lookup = AddressStatsLookup::default(); + let tx = self.tx.clone(); tokio::task::spawn(async move { let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD / 15)); loop { interval.tick().await; - let address_count = get_number_of_addresses(); - for address_id in 0..address_count { + for ((pool_name, username), _pool) in get_all_pools() { let _ = tx.try_send(Event { - name: EventName::UpdateStats, + name: EventName::UpdateStats { + pool_name, + username, + }, value: 0, - process_id: -1, - address_id: address_id, }); } } @@ -370,13 +538,10 @@ impl Collector { tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD)); loop { interval.tick().await; - let address_count = get_number_of_addresses(); - for address_id in 0..address_count { + for address_id in 0..get_number_of_addresses() { let _ = tx.try_send(Event { - name: EventName::UpdateAverages, + name: EventName::UpdateAverages { address_id }, value: 0, - process_id: -1, - address_id: address_id, }); } } @@ -392,119 +557,328 @@ impl Collector { } }; - let stats = stats - .entry(stat.address_id) - .or_insert(stats_template.clone()); - let client_server_states = client_server_states - .entry(stat.address_id) - .or_insert(HashMap::new()); - let old_stats = old_stats.entry(stat.address_id).or_insert(HashMap::new()); - // Some are counters, some are gauges... match stat.name { - EventName::Query => { - let counter = stats.entry("total_query_count").or_insert(0); - *counter += stat.value; + EventName::Query { + client_id, + server_id, + } => { + // Update client stats + let app_name = match client_states.get_mut(&client_id) { + Some(client_info) => { + client_info.query_count += stat.value as u64; + client_info.application_name.to_string() + } + None => String::from("Undefined"), + }; + + // Update server stats and pool aggergation stats + match server_states.get_mut(&server_id) { + Some(server_info) => { + server_info.query_count += stat.value as u64; + server_info.application_name = app_name; + + let pool_stats = address_stat_lookup + .entry(server_info.address_id) + .or_insert(HashMap::default()); + let counter = pool_stats + .entry("total_query_count".to_string()) + .or_insert(0); + *counter += stat.value; + } + None => (), + } } - EventName::Transaction => { - let counter = stats.entry("total_xact_count").or_insert(0); - *counter += stat.value; + EventName::Transaction { + client_id, + server_id, + } => { + // Update client stats + let app_name = match client_states.get_mut(&client_id) { + Some(client_info) => { + client_info.transaction_count += stat.value as u64; + client_info.application_name.to_string() + } + None => String::from("Undefined"), + }; + + // Update server stats and pool aggergation stats + match server_states.get_mut(&server_id) { + Some(server_info) => { + server_info.transaction_count += stat.value as u64; + server_info.application_name = app_name; + + let address_stats = address_stat_lookup + .entry(server_info.address_id) + .or_insert(HashMap::default()); + let counter = address_stats + .entry("total_xact_count".to_string()) + .or_insert(0); + *counter += stat.value; + } + None => (), + } } - EventName::DataSent => { - let counter = stats.entry("total_sent").or_insert(0); - *counter += stat.value; + EventName::DataSentToServer { server_id } => { + // Update server stats and address aggergation stats + match server_states.get_mut(&server_id) { + Some(server_info) => { + server_info.bytes_sent += stat.value as u64; + + let address_stats = address_stat_lookup + .entry(server_info.address_id) + .or_insert(HashMap::default()); + let counter = + address_stats.entry("total_sent".to_string()).or_insert(0); + *counter += stat.value; + } + None => (), + } } - EventName::DataReceived => { - let counter = stats.entry("total_received").or_insert(0); - *counter += stat.value; + EventName::DataReceivedFromServer { server_id } => { + // Update server states and address aggergation stats + match server_states.get_mut(&server_id) { + Some(server_info) => { + server_info.bytes_received += stat.value as u64; + + let address_stats = address_stat_lookup + .entry(server_info.address_id) + .or_insert(HashMap::default()); + let counter = address_stats + .entry("total_received".to_string()) + .or_insert(0); + *counter += stat.value; + } + None => (), + } } - EventName::CheckoutTime => { - let counter = stats.entry("total_wait_time").or_insert(0); - *counter += stat.value; + EventName::CheckoutTime { + client_id, + server_id, + } => { + // Update client stats + let app_name = match client_states.get_mut(&client_id) { + Some(client_info) => { + client_info.total_wait_time += stat.value as u64; + client_info.application_name.to_string() + } + None => String::from("Undefined"), + }; + + // Update server stats and address aggergation stats + match server_states.get_mut(&server_id) { + Some(server_info) => { + server_info.application_name = app_name; + + let pool_stats = address_stat_lookup + .entry(server_info.address_id) + .or_insert(HashMap::default()); + let counter = + pool_stats.entry("total_wait_time".to_string()).or_insert(0); + *counter += stat.value; + + let counter = pool_stats.entry("maxwait_us".to_string()).or_insert(0); + let mic_part = stat.value % 1_000_000; + + // Report max time here + if mic_part > *counter { + *counter = mic_part; + } - let counter = stats.entry("maxwait_us").or_insert(0); - let mic_part = stat.value % 1_000_000; + let counter = pool_stats.entry("maxwait".to_string()).or_insert(0); + let seconds = *counter / 1_000_000; - // Report max time here - if mic_part > *counter { - *counter = mic_part; + if seconds > *counter { + *counter = seconds; + } + } + None => (), } + } - let counter = stats.entry("maxwait").or_insert(0); - let seconds = *counter / 1_000_000; + EventName::ClientRegistered { + client_id, + pool_name, + username, + application_name, + } => { + match client_states.get_mut(&client_id) { + Some(_) => warn!("Client {:?} was double registered!", client_id), + None => { + client_states.insert( + client_id, + ClientInformation { + state: ClientState::Idle, + connect_time: Instant::now(), + client_id, + pool_name: pool_name.clone(), + username: username.clone(), + application_name: application_name.clone(), + total_wait_time: 0, + transaction_count: 0, + query_count: 0, + error_count: 0, + }, + ); + } + }; + } - if seconds > *counter { - *counter = seconds; + EventName::ClientBanError { + client_id, + address_id, + } => { + match client_states.get_mut(&client_id) { + Some(client_info) => client_info.error_count += stat.value as u64, + None => warn!("Got event {:?} for unregistered client", stat.name), } + + // Update address aggregation stats + let address_stats = address_stat_lookup + .entry(address_id) + .or_insert(HashMap::default()); + let counter = address_stats.entry("total_errors".to_string()).or_insert(0); + *counter += stat.value; } - EventName::ClientActive - | EventName::ClientWaiting - | EventName::ClientIdle - | EventName::ServerActive - | EventName::ServerIdle - | EventName::ServerTested - | EventName::ServerLogin => { - client_server_states.insert(stat.process_id, stat.name); + EventName::ClientCheckoutError { + client_id, + address_id, + } => { + match client_states.get_mut(&client_id) { + Some(client_info) => client_info.error_count += stat.value as u64, + None => warn!("Got event {:?} for unregistered client", stat.name), + } + + // Update address aggregation stats + let address_stats = address_stat_lookup + .entry(address_id) + .or_insert(HashMap::default()); + let counter = address_stats.entry("total_errors".to_string()).or_insert(0); + *counter += stat.value; } - EventName::ClientDisconnecting | EventName::ServerDisconnecting => { - client_server_states.remove(&stat.process_id); + EventName::ClientIdle { client_id } => { + match client_states.get_mut(&client_id) { + Some(client_state) => client_state.state = ClientState::Idle, + None => warn!("Got event {:?} for unregistered client", stat.name), + }; } - EventName::UpdateStats => { - // Calculate connection states - for (_, state) in client_server_states.iter() { - match state { - EventName::ClientActive => { - let counter = stats.entry("cl_active").or_insert(0); - *counter += 1; - } + EventName::ClientWaiting { client_id } => { + match client_states.get_mut(&client_id) { + Some(client_state) => client_state.state = ClientState::Waiting, + None => warn!("Got event {:?} for unregistered client", stat.name), + }; + } - EventName::ClientWaiting => { - let counter = stats.entry("cl_waiting").or_insert(0); - *counter += 1; - } + EventName::ClientActive { + client_id, + server_id: _, + } => { + match client_states.get_mut(&client_id) { + Some(client_state) => client_state.state = ClientState::Active, + None => warn!("Got event {:?} for unregistered client", stat.name), + }; + } - EventName::ServerIdle => { - let counter = stats.entry("sv_idle").or_insert(0); - *counter += 1; - } + EventName::ClientDisconnecting { client_id } => { + client_states.remove(&client_id); + } - EventName::ServerActive => { - let counter = stats.entry("sv_active").or_insert(0); - *counter += 1; - } + EventName::ServerRegistered { + address_name, + server_id, + address_id, + pool_name, + username, + } => { + server_states.insert( + server_id, + ServerInformation { + address_id, + address_name, + server_id, + username, + pool_name, + + state: ServerState::Idle, + application_name: String::from("Undefined"), + connect_time: Instant::now(), + bytes_sent: 0, + bytes_received: 0, + transaction_count: 0, + query_count: 0, + error_count: 0, + }, + ); + } - EventName::ServerTested => { - let counter = stats.entry("sv_tested").or_insert(0); - *counter += 1; - } + EventName::ServerLogin { server_id } => { + match server_states.get_mut(&server_id) { + Some(server_state) => { + server_state.state = ServerState::Login; + server_state.application_name = String::from("Undefined"); + } + None => warn!("Got event {:?} for unregistered server", stat.name), + }; + } - EventName::ServerLogin => { - let counter = stats.entry("sv_login").or_insert(0); - *counter += 1; - } + EventName::ServerTested { server_id } => { + match server_states.get_mut(&server_id) { + Some(server_state) => { + server_state.state = ServerState::Tested; + server_state.application_name = String::from("Undefined"); + } + None => warn!("Got event {:?} for unregistered server", stat.name), + }; + } - EventName::ClientIdle => { - let counter = stats.entry("cl_idle").or_insert(0); - *counter += 1; - } + EventName::ServerIdle { server_id } => { + match server_states.get_mut(&server_id) { + Some(server_state) => { + server_state.state = ServerState::Idle; + server_state.application_name = String::from("Undefined"); + } + None => warn!("Got event {:?} for unregistered server", stat.name), + }; + } - _ => unreachable!(), - }; - } + EventName::ServerActive { + client_id, + server_id, + } => { + // Update client stats + let app_name = match client_states.get_mut(&client_id) { + Some(client_info) => client_info.application_name.to_string(), + None => String::from("Undefined"), + }; + + // Update server stats + match server_states.get_mut(&server_id) { + Some(server_state) => { + server_state.state = ServerState::Active; + server_state.application_name = app_name; + } + None => warn!("Got event {:?} for unregistered server", stat.name), + }; + } - // Update latest stats used in SHOW STATS - let mut guard = LATEST_STATS.lock(); - for (key, value) in stats.iter() { - let entry = guard.entry(stat.address_id).or_insert(HashMap::new()); - entry.insert(key.to_string(), value.clone()); - } + EventName::ServerDisconnecting { server_id } => { + server_states.remove(&server_id); + } + + EventName::UpdateStats { + pool_name, + username, + } => { + let pool_stats = pool_stat_lookup + .entry((pool_name.clone(), username.clone())) + .or_insert(HashMap::default()); // These are re-calculated every iteration of the loop, so we don't want to add values // from the last iteration. @@ -519,17 +893,79 @@ impl Collector { "maxwait", "maxwait_us", ] { - stats.insert(stat, 0); + pool_stats.insert(stat.to_string(), 0); } + + for (_, client_info) in client_states.iter() { + if client_info.pool_name != pool_name || client_info.username != username { + continue; + } + match client_info.state { + ClientState::Idle => { + let counter = pool_stats.entry("cl_idle".to_string()).or_insert(0); + *counter += 1; + } + ClientState::Waiting => { + let counter = + pool_stats.entry("cl_waiting".to_string()).or_insert(0); + *counter += 1; + } + ClientState::Active => { + let counter = + pool_stats.entry("cl_active".to_string()).or_insert(0); + *counter += 1; + } + }; + } + + for (_, server_info) in server_states.iter() { + if server_info.pool_name != pool_name || server_info.username != username { + continue; + } + match server_info.state { + ServerState::Login => { + let counter = pool_stats.entry("sv_login".to_string()).or_insert(0); + *counter += 1; + } + ServerState::Tested => { + let counter = + pool_stats.entry("sv_tested".to_string()).or_insert(0); + *counter += 1; + } + ServerState::Active => { + let counter = + pool_stats.entry("sv_active".to_string()).or_insert(0); + *counter += 1; + } + ServerState::Idle => { + let counter = pool_stats.entry("sv_idle".to_string()).or_insert(0); + *counter += 1; + } + }; + } + + // The following calls publish the internal stats making it visible + // to clients using admin database to issue queries like `SHOW STATS` + LATEST_CLIENT_STATS.store(Arc::new(client_states.clone())); + LATEST_SERVER_STATS.store(Arc::new(server_states.clone())); + LATEST_POOL_STATS.store(Arc::new(pool_stat_lookup.clone())); } - EventName::UpdateAverages => { + EventName::UpdateAverages { address_id } => { + let stats = address_stat_lookup + .entry(address_id) + .or_insert(HashMap::default()); + let old_stats = address_old_stat_lookup + .entry(address_id) + .or_insert(HashMap::default()); + // Calculate averages for stat in &[ "avg_query_count", "avg_query_time", "avg_recv", "avg_sent", + "avg_errors", "avg_xact_time", "avg_xact_count", "avg_wait_time", @@ -543,19 +979,38 @@ impl Collector { let new_value = stats.get(total_name.as_str()).unwrap_or(&0).to_owned(); let avg = (new_value - *old_value) / (STAT_PERIOD as i64 / 1_000); // Avg / second - stats.insert(stat, avg); + stats.insert(stat.to_string(), avg); *old_value = new_value; } + LATEST_ADDRESS_STATS.store(Arc::new(address_stat_lookup.clone())); } }; } } } -/// Get a snapshot of statistics. Updated once a second +/// Get a snapshot of client statistics. Updated once a second +/// by the `Collector`. +pub fn get_client_stats() -> ClientStatesLookup { + (*(*LATEST_CLIENT_STATS.load())).clone() +} + +/// Get a snapshot of server statistics. Updated once a second +/// by the `Collector`. +pub fn get_server_stats() -> ServerStatesLookup { + (*(*LATEST_SERVER_STATS.load())).clone() +} + +/// Get a snapshot of pool statistics. Updated once a second +/// by the `Collector`. +pub fn get_pool_stats() -> PoolStatsLookup { + (*(*LATEST_POOL_STATS.load())).clone() +} + +/// Get a snapshot of address statistics. Updated once a second /// by the `Collector`. -pub fn get_stats() -> HashMap> { - LATEST_STATS.lock().clone() +pub fn get_address_stats() -> AddressStatsLookup { + (*(*LATEST_ADDRESS_STATS.load())).clone() } /// Get the statistics reporter used to update stats across the pools/clients. diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb new file mode 100644 index 000000000..b57098455 --- /dev/null +++ b/tests/ruby/admin_spec.rb @@ -0,0 +1,214 @@ +# frozen_string_literal: true +require 'uri' +require_relative 'spec_helper' + +describe "Admin" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 10) } + let(:pgcat_conn_str) { processes.pgcat.connection_string("sharded_db", "sharding_user") } + + after do + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + + describe "SHOW POOLS" do + context "bad credentials" do + it "does not change any stats" do + bad_passsword_url = URI(pgcat_conn_str) + bad_passsword_url.password = "wrong" + expect { PG::connect("#{bad_passsword_url.to_s}?application_name=bad_password") }.to raise_error(PG::ConnectionBad) + + sleep(1) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + + expect(results["sv_idle"]).to eq("1") + end + end + + context "bad database name" do + it "does not change any stats" do + bad_db_url = URI(pgcat_conn_str) + bad_db_url.path = "/wrong_db" + expect { PG::connect("#{bad_db_url.to_s}?application_name=bad_db") }.to raise_error(PG::ConnectionBad) + + sleep(1) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + + expect(results["sv_idle"]).to eq("1") + end + end + + context "client connects but issues no queries" do + it "only affects cl_idle stats" do + connections = Array.new(20) { PG::connect(pgcat_conn_str) } + sleep(1) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_idle"]).to eq("20") + expect(results["sv_idle"]).to eq("1") + + connections.map(&:close) + sleep(1.1) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_idle cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["sv_idle"]).to eq("1") + end + end + + context "clients connect and make one query" do + it "only affects cl_idle, sv_idle stats" do + connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + Thread.new { c.async_exec("SELECT pg_sleep(2.5)") } + end + + sleep(1.1) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_waiting cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_active"]).to eq("5") + expect(results["sv_active"]).to eq("5") + + sleep(3) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_idle"]).to eq("5") + expect(results["sv_idle"]).to eq("5") + + connections.map(&:close) + sleep(1) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["sv_idle"]).to eq("5") + end + end + + context "client connects and opens a transaction and closes connection uncleanly" do + it "produces correct statistics" do + connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + Thread.new do + c.async_exec("BEGIN") + c.async_exec("SELECT pg_sleep(0.01)") + c.close + end + end + + sleep(1.1) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["sv_idle"]).to eq("5") + end + end + + context "clients overwhelm server pools" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) } + + it "cl_waiting is updated to show it" do + threads = [] + connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") } + end + + sleep(1.1) # Allow time for stats to update + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + + expect(results["cl_waiting"]).to eq("2") + expect(results["cl_active"]).to eq("2") + expect(results["sv_active"]).to eq("2") + + sleep(2.5) # Allow time for stats to update + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_idle"]).to eq("4") + expect(results["sv_idle"]).to eq("2") + + threads.map(&:join) + connections.map(&:close) + end + end + end + + describe "SHOW CLIENTS" do + it "reports correct number and application names" do + conn_str = processes.pgcat.connection_string("sharded_db", "sharding_user") + connections = Array.new(20) { |i| PG::connect("#{conn_str}?application_name=app#{i % 5}") } + + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + sleep(1) # Wait for stats to be updated + + results = admin_conn.async_exec("SHOW CLIENTS") + expect(results.count).to eq(21) # count admin clients + expect(results.select { |c| c["application_name"] == "app3" || c["application_name"] == "app4" }.count).to eq(8) + expect(results.select { |c| c["database"] == "pgcat" }.count).to eq(1) + + connections[0..5].map(&:close) + sleep(1) # Wait for stats to be updated + results = admin_conn.async_exec("SHOW CLIENTS") + expect(results.count).to eq(15) + + connections[6..].map(&:close) + sleep(1) # Wait for stats to be updated + expect(admin_conn.async_exec("SHOW CLIENTS").count).to eq(1) + admin_conn.close + end + + it "reports correct number of queries and transactions" do + conn_str = processes.pgcat.connection_string("sharded_db", "sharding_user") + + connections = Array.new(2) { |i| PG::connect("#{conn_str}?application_name=app#{i}") } + connections.each do |c| + c.async_exec("SELECT 1") + c.async_exec("SELECT 2") + c.async_exec("SELECT 3") + c.async_exec("BEGIN") + c.async_exec("SELECT 4") + c.async_exec("SELECT 5") + c.async_exec("COMMIT") + end + + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + sleep(1) # Wait for stats to be updated + + results = admin_conn.async_exec("SHOW CLIENTS") + expect(results.count).to eq(3) + normal_client_results = results.reject { |r| r["database"] == "pgcat" } + expect(normal_client_results[0]["transaction_count"]).to eq("4") + expect(normal_client_results[1]["transaction_count"]).to eq("4") + expect(normal_client_results[0]["query_count"]).to eq("7") + expect(normal_client_results[1]["query_count"]).to eq("7") + + admin_conn.close + connections.map(&:close) + end + end +end diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index 80ac9dab6..55847ed65 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -46,6 +46,50 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction") end end + def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction") + user = { + "password" => "sharding_user", + "pool_size" => pool_size, + "statement_timeout" => 0, + "username" => "sharding_user" + } + + pgcat = PgcatProcess.new("trace") + pgcat_cfg = pgcat.current_config + + primary = PgInstance.new(5432, user["username"], user["password"], "shard0") + + # Main proxy configs + pgcat_cfg["pools"] = { + "#{pool_name}" => { + "default_role" => "primary", + "pool_mode" => pool_mode, + "primary_reads_enabled" => false, + "query_parser_enabled" => false, + "sharding_function" => "pg_bigint_hash", + "shards" => { + "0" => { + "database" => "shard0", + "servers" => [ + ["localhost", primary.port.to_s, "primary"] + ] + }, + }, + "users" => { "0" => user } + } + } + pgcat_cfg["general"]["port"] = pgcat.port + pgcat.update_config(pgcat_cfg) + pgcat.start + pgcat.wait_until_ready + + OpenStruct.new.tap do |struct| + struct.pgcat = pgcat + struct.primary = primary + struct.all_databases = [primary] + end + end + def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction") user = { "password" => "sharding_user", diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index a5a6d3d38..b6e798aed 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -111,6 +111,6 @@ def example_connection_string username = cfg["pools"][first_pool_name]["users"]["0"]["username"] password = cfg["pools"][first_pool_name]["users"]["0"]["password"] - "postgresql://#{username}:#{password}@0.0.0.0:#{@port}/#{db_name}" + "postgresql://#{username}:#{password}@0.0.0.0:#{@port}/#{db_name}?application_name=example_app" end end From 4151c6bbb8d2dca424897acde32a41abbae47ea2 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 15 Sep 2022 01:21:45 -0500 Subject: [PATCH 209/551] Report Query times (#166) * Report avg and total query timing * Report query times * fmt --- src/client.rs | 8 +++++++- src/stats.rs | 11 +++++++++-- tests/ruby/admin_spec.rb | 23 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/client.rs b/src/client.rs index c07a21f06..3b0b0ea71 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2,6 +2,7 @@ use bytes::{Buf, BufMut, BytesMut}; use log::{debug, error, info, trace}; use std::collections::HashMap; +use std::time::Instant; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; use tokio::net::TcpStream; use tokio::sync::broadcast::Receiver; @@ -994,6 +995,7 @@ where self.send_server_message(server, message, &address, &pool) .await?; + let query_start = Instant::now(); // Read all data the server has to offer, which can be multiple messages // buffered in 8196 bytes chunks. loop { @@ -1013,7 +1015,11 @@ where } // Report query executed statistics. - self.stats.query(self.process_id, server.server_id()); + self.stats.query( + self.process_id, + server.server_id(), + Instant::now().duration_since(query_start).as_millis(), + ); Ok(()) } diff --git a/src/stats.rs b/src/stats.rs index 9ab71677c..90b55ddd8 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -9,7 +9,6 @@ use tokio::sync::mpsc::{channel, Receiver, Sender}; use tokio::time::Instant; use crate::pool::{get_all_pools, get_number_of_addresses}; -use crate::server; /// Convenience types for various stats type ClientStatesLookup = HashMap; @@ -138,6 +137,7 @@ enum EventName { Query { client_id: i32, server_id: i32, + duration_ms: u128, }, Transaction { client_id: i32, @@ -269,11 +269,12 @@ impl Reporter { } /// Report a query executed by a client against a server - pub fn query(&self, client_id: i32, server_id: i32) { + pub fn query(&self, client_id: i32, server_id: i32, duration_ms: u128) { let event = Event { name: EventName::Query { client_id, server_id, + duration_ms, }, value: 1, }; @@ -562,6 +563,7 @@ impl Collector { EventName::Query { client_id, server_id, + duration_ms, } => { // Update client stats let app_name = match client_states.get_mut(&client_id) { @@ -585,6 +587,11 @@ impl Collector { .entry("total_query_count".to_string()) .or_insert(0); *counter += stat.value; + + let duration = pool_stats + .entry("total_query_time".to_string()) + .or_insert(0); + *duration += duration_ms as i64; } None => (), } diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index b57098455..a348146ef 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -11,6 +11,29 @@ processes.pgcat.shutdown end + describe "SHOW STATS" do + context "clients connect and make one query" do + it "updates *_query_time and *_wait_time" do + connection = PG::connect("#{pgcat_conn_str}?application_name=one_query") + connection.async_exec("SELECT pg_sleep(0.25)") + connection.async_exec("SELECT pg_sleep(0.25)") + connection.async_exec("SELECT pg_sleep(0.25)") + connection.close + + # wait for averages to be calculated, we shouldn't do this too often + sleep(15.5) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW STATS")[0] + admin_conn.close + expect(results["total_query_time"].to_i).to be_within(200).of(750) + expect(results["avg_query_time"].to_i).to_not eq(0) + + expect(results["total_wait_time"].to_i).to_not eq(0) + expect(results["avg_wait_time"].to_i).to_not eq(0) + end + end + end + describe "SHOW POOLS" do context "bad credentials" do it "does not change any stats" do From efa769a2f8c26fdbfe426d98aaad9733118eef86 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Tue, 20 Sep 2022 21:47:32 -0400 Subject: [PATCH 210/551] Export pgcat objects in lib (#169) * Export pgcat objects in lib * fmt --- src/admin.rs | 10 +++++----- src/config.rs | 2 +- src/lib.rs | 32 ++++++++++++++++++++++++++++++++ src/main.rs | 23 +---------------------- src/messages.rs | 2 +- src/server.rs | 2 +- 6 files changed, 41 insertions(+), 30 deletions(-) create mode 100644 src/lib.rs diff --git a/src/admin.rs b/src/admin.rs index ed2d3de3a..1aa2bced1 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -16,11 +16,11 @@ use crate::ClientServerMap; pub fn generate_server_info_for_admin() -> BytesMut { let mut server_info = BytesMut::new(); - server_info.put(server_paramater_message("application_name", "")); - server_info.put(server_paramater_message("client_encoding", "UTF8")); - server_info.put(server_paramater_message("server_encoding", "UTF8")); - server_info.put(server_paramater_message("server_version", VERSION)); - server_info.put(server_paramater_message("DateStyle", "ISO, MDY")); + server_info.put(server_parameter_message("application_name", "")); + server_info.put(server_parameter_message("client_encoding", "UTF8")); + server_info.put(server_parameter_message("server_encoding", "UTF8")); + server_info.put(server_parameter_message("server_version", VERSION)); + server_info.put(server_parameter_message("DateStyle", "ISO, MDY")); return server_info; } diff --git a/src/config.rs b/src/config.rs index 5c1226117..9ef4d1783 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,8 +12,8 @@ use tokio::io::AsyncReadExt; use toml; use crate::errors::Error; +use crate::pool::{ClientServerMap, ConnectionPool}; use crate::tls::{load_certs, load_keys}; -use crate::{ClientServerMap, ConnectionPool}; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 000000000..e9a683f3d --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,32 @@ +pub mod config; +pub mod constants; +pub mod errors; +pub mod messages; +pub mod pool; +pub mod scram; +pub mod server; +pub mod sharding; +pub mod stats; +pub mod tls; + +/// Format chrono::Duration to be more human-friendly. +/// +/// # Arguments +/// +/// * `duration` - A duration of time +pub fn format_duration(duration: &chrono::Duration) -> String { + let milliseconds = format!("{:0>3}", duration.num_milliseconds() % 1000); + + let seconds = format!("{:0>2}", duration.num_seconds() % 60); + + let minutes = format!("{:0>2}", duration.num_minutes() % 60); + + let hours = format!("{:0>2}", duration.num_hours() % 24); + + let days = duration.num_days().to_string(); + + format!( + "{}d {}:{}:{}.{}", + days, hours, minutes, seconds, milliseconds + ) +} diff --git a/src/main.rs b/src/main.rs index 0d4bd37ab..6690b568e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -39,6 +39,7 @@ extern crate toml; use log::{debug, error, info}; use parking_lot::Mutex; +use pgcat::format_duration; use tokio::net::TcpListener; use tokio::{ signal::unix::{signal as unix_signal, SignalKind}, @@ -301,25 +302,3 @@ async fn main() { info!("Shutting down..."); } - -/// Format chrono::Duration to be more human-friendly. -/// -/// # Arguments -/// -/// * `duration` - A duration of time -fn format_duration(duration: &chrono::Duration) -> String { - let milliseconds = format!("{:0>3}", duration.num_milliseconds() % 1000); - - let seconds = format!("{:0>2}", duration.num_seconds() % 60); - - let minutes = format!("{:0>2}", duration.num_minutes() % 60); - - let hours = format!("{:0>2}", duration.num_hours() % 24); - - let days = duration.num_days().to_string(); - - format!( - "{}d {}:{}:{}.{}", - days, hours, minutes, seconds, milliseconds - ) -} diff --git a/src/messages.rs b/src/messages.rs index 113e1ed52..78cb9dbf7 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -496,7 +496,7 @@ where Ok(bytes) } -pub fn server_paramater_message(key: &str, value: &str) -> BytesMut { +pub fn server_parameter_message(key: &str, value: &str) -> BytesMut { let mut server_info = BytesMut::new(); let null_byte_size = 1; diff --git a/src/server.rs b/src/server.rs index d8d23d6c7..64886be51 100644 --- a/src/server.rs +++ b/src/server.rs @@ -14,9 +14,9 @@ use crate::config::{Address, User}; use crate::constants::*; use crate::errors::Error; use crate::messages::*; +use crate::pool::ClientServerMap; use crate::scram::ScramSha256; use crate::stats::Reporter; -use crate::ClientServerMap; /// Server state. pub struct Server { From 6a9d318b1b6c6c0d9d8b6db2b38aa2d1fff21917 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 21 Sep 2022 16:48:33 -0400 Subject: [PATCH 211/551] Update to latest library versions (#170) --- Cargo.lock | 106 ++++++++++++++++++++++++++++++++++++++++------------ Cargo.toml | 8 ++-- src/pool.rs | 2 +- 3 files changed, 87 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af4826b87..2bced0610 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,14 +53,14 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bb8" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9f4fa9768efd269499d8fba693260cfc670891cf6de3adc935588447a77cc8" +checksum = "1627eccf3aa91405435ba240be23513eeca466b5dc33866422672264de061582" dependencies = [ "async-trait", "futures-channel", "futures-util", - "parking_lot", + "parking_lot 0.12.1", "tokio", ] @@ -493,7 +493,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.3", ] [[package]] @@ -510,6 +520,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "pgcat" version = "0.6.0-alpha1" @@ -528,7 +551,7 @@ dependencies = [ "md-5", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "phf", "rand", "regex", @@ -546,20 +569,19 @@ dependencies = [ [[package]] name = "phf" -version = "0.10.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" dependencies = [ "phf_macros", "phf_shared", - "proc-macro-hack", ] [[package]] name = "phf_generator" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" dependencies = [ "phf_shared", "rand", @@ -567,13 +589,12 @@ dependencies = [ [[package]] name = "phf_macros" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" +checksum = "92aacdc5f16768709a569e913f7451034034178b05bdc8acda226659a3dccc66" dependencies = [ "phf_generator", "phf_shared", - "proc-macro-hack", "proc-macro2", "quote", "syn", @@ -581,9 +602,9 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" dependencies = [ "siphasher", ] @@ -606,12 +627,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - [[package]] name = "proc-macro2" version = "1.0.36" @@ -832,9 +847,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.14.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8f192f29f4aa49e57bebd0aa05858e0a1f32dd270af36efe49edb82cbfffab6" +checksum = "0beb13adabbdda01b63d595f38c8bfd19a361e697fd94ce0098a634077bc5b25" dependencies = [ "log", ] @@ -913,7 +928,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.11.2", "pin-project-lite", "signal-hook-registry", "tokio-macros", @@ -1168,3 +1183,46 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" diff --git a/Cargo.toml b/Cargo.toml index 226a20d6c..48ba966d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" tokio = { version = "1", features = ["full"] } bytes = "1" md-5 = "0.10" -bb8 = "0.7" +bb8 = "0.8.0" async-trait = "0.1" rand = "0.8" chrono = "0.4" @@ -20,11 +20,11 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = "0.14" +sqlparser = "0.23.0" log = "0.4" arc-swap = "1" env_logger = "0.9" -parking_lot = "0.11" +parking_lot = "0.12.1" hmac = "0.12" sha2 = "0.10" base64 = "0.13" @@ -32,5 +32,5 @@ stringprep = "0.1" tokio-rustls = "0.23" rustls-pemfile = "1" hyper = { version = "0.14", features = ["full"] } -phf = { version = "0.10", features = ["macros"] } +phf = { version = "0.11.1", features = ["macros"] } exitcode = "1.1.2" diff --git a/src/pool.rs b/src/pool.rs index 34af354a4..c4a424e13 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -595,7 +595,7 @@ impl ManageConnection for ServerPool { } /// Determines if the connection is still connected to the database. - async fn is_valid(&self, _conn: &mut PooledConnection<'_, Self>) -> Result<(), Self::Error> { + async fn is_valid(&self, _conn: &mut Self::Connection) -> Result<(), Self::Error> { Ok(()) } From 2be0c89c3ff3f8cfb03bff0b9755e31b2ee866c6 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 22 Sep 2022 13:07:02 -0400 Subject: [PATCH 212/551] Minor refactor for configs (#172) * Changes shard struct to use vector of ServerConfig * Adds to query router * Change client disconnect with error message to warn instead of debug * Add warning logs for clean up actions --- src/config.rs | 44 ++++++++++++++++++++++---------------------- src/main.rs | 4 ++-- src/pool.rs | 17 ++++------------- src/query_router.rs | 4 ++-- src/server.rs | 2 ++ 5 files changed, 32 insertions(+), 39 deletions(-) diff --git a/src/config.rs b/src/config.rs index 9ef4d1783..e4c25a241 100644 --- a/src/config.rs +++ b/src/config.rs @@ -23,7 +23,9 @@ static CONFIG: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Config /// Server role: primary or replica. #[derive(Clone, PartialEq, Serialize, Deserialize, Hash, std::cmp::Eq, Debug, Copy)] pub enum Role { + #[serde(alias = "primary", alias = "Primary")] Primary, + #[serde(alias = "replica", alias = "Replica")] Replica, } @@ -202,17 +204,28 @@ impl Default for Pool { } } +#[derive(Clone, PartialEq, Serialize, Deserialize, Debug, Hash, Eq)] +pub struct ServerConfig { + pub host: String, + pub port: u16, + pub role: Role, +} + /// Shard configuration. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Shard { pub database: String, - pub servers: Vec<(String, u16, String)>, + pub servers: Vec, } impl Default for Shard { fn default() -> Shard { Shard { - servers: vec![(String::from("localhost"), 5432, String::from("primary"))], + servers: vec![ServerConfig { + host: String::from("localhost"), + port: 5432, + role: Role::Primary, + }], database: String::from("postgres"), } } @@ -538,23 +551,10 @@ pub async fn parse(path: &str) -> Result<(), Error> { dup_check.insert(server); // Check that we define only zero or one primary. - match server.2.as_ref() { - "primary" => primary_count += 1, + match server.role { + Role::Primary => primary_count += 1, _ => (), }; - - // Check role spelling. - match server.2.as_ref() { - "primary" => (), - "replica" => (), - _ => { - error!( - "Shard {} server role must be either 'primary' or 'replica', got: '{}'", - shard.0, server.2 - ); - return Err(Error::BadConfig); - } - }; } if primary_count > 1 { @@ -617,12 +617,12 @@ mod test { assert_eq!(get_config().pools["simple_db"].users.len(), 1); assert_eq!( - get_config().pools["sharded_db"].shards["0"].servers[0].0, + get_config().pools["sharded_db"].shards["0"].servers[0].host, "127.0.0.1" ); assert_eq!( - get_config().pools["sharded_db"].shards["1"].servers[0].2, - "primary" + get_config().pools["sharded_db"].shards["1"].servers[0].role, + Role::Primary ); assert_eq!( get_config().pools["sharded_db"].shards["1"].database, @@ -640,11 +640,11 @@ mod test { assert_eq!(get_config().pools["sharded_db"].default_role, "any"); assert_eq!( - get_config().pools["simple_db"].shards["0"].servers[0].0, + get_config().pools["simple_db"].shards["0"].servers[0].host, "127.0.0.1" ); assert_eq!( - get_config().pools["simple_db"].shards["0"].servers[0].1, + get_config().pools["simple_db"].shards["0"].servers[0].port, 5432 ); assert_eq!( diff --git a/src/main.rs b/src/main.rs index 6690b568e..467527c97 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,7 +37,7 @@ extern crate tokio; extern crate tokio_rustls; extern crate toml; -use log::{debug, error, info}; +use log::{error, info, warn}; use parking_lot::Mutex; use pgcat::format_duration; use tokio::net::TcpListener; @@ -279,7 +279,7 @@ async fn main() { } } - debug!("Client disconnected with error {:?}", err); + warn!("Client disconnected with error {:?}", err); } }; }); diff --git a/src/pool.rs b/src/pool.rs index c4a424e13..f81a1e075 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -143,21 +143,12 @@ impl ConnectionPool { let mut replica_number = 0; for server in shard.servers.iter() { - let role = match server.2.as_ref() { - "primary" => Role::Primary, - "replica" => Role::Replica, - _ => { - error!("Config error: server role can be 'primary' or 'replica', have: '{}'. Defaulting to 'replica'.", server.2); - Role::Replica - } - }; - let address = Address { id: address_id, database: shard.database.clone(), - host: server.0.clone(), - port: server.1 as u16, - role: role, + host: server.host.clone(), + port: server.port, + role: server.role, address_index, replica_number, shard: shard_idx.parse::().unwrap(), @@ -168,7 +159,7 @@ impl ConnectionPool { address_id += 1; address_index += 1; - if role == Role::Replica { + if server.role == Role::Replica { replica_number += 1; } diff --git a/src/query_router.rs b/src/query_router.rs index f9d5f0b37..85d4f8b4e 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -169,8 +169,8 @@ impl QueryRouter { Command::ShowShard => self.shard().to_string(), Command::ShowServerRole => match self.active_role { - Some(Role::Primary) => String::from("primary"), - Some(Role::Replica) => String::from("replica"), + Some(Role::Primary) => Role::Primary.to_string(), + Some(Role::Replica) => Role::Replica.to_string(), None => { if self.query_parser_enabled { String::from("auto") diff --git a/src/server.rs b/src/server.rs index 64886be51..dbac9bc09 100644 --- a/src/server.rs +++ b/src/server.rs @@ -591,6 +591,7 @@ impl Server { // server connection thrashing if clients repeatedly do this. // Instead, we ROLLBACK that transaction before putting the connection back in the pool if self.in_transaction() { + warn!("Server returned while still in transaction, rolling back transaction"); self.query("ROLLBACK").await?; } @@ -600,6 +601,7 @@ impl Server { // send `DISCARD ALL` if we think the session is altered instead of just sending // it before each checkin. if self.needs_cleanup { + warn!("Server returned with session state altered, discarding state"); self.query("DISCARD ALL").await?; self.needs_cleanup = false; } From 1e9ad631c7860ceebd3ee75486fd579342de3f98 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Fri, 23 Sep 2022 02:00:46 -0400 Subject: [PATCH 213/551] Add defaults for configs (#174) * add statement timeout to readme * Add defaults to various configs * primary read enabled default to false --- README.md | 2 + src/client.rs | 4 +- src/config.rs | 139 ++++++++++++++++++++++++++++++++++---------- src/pool.rs | 26 +-------- src/query_router.rs | 2 +- 5 files changed, 115 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index dfecc0abb..1e77bafb2 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ psql -h 127.0.0.1 -p 6432 -c 'SELECT 1' | **`user`** | | | | `name` | The user name. | `sharding_user` | | `password` | The user password in plaintext. | `hunter2` | +| `statement_timeout` | Timeout in milliseconds for how long a query takes to execute | `0 (disabled)` | + | | | | | **`shards`** | Shards are numerically numbered starting from 0; the order in the config is preserved by the pooler to route queries accordingly. | `[shards.0]` | | `servers` | List of servers to connect to and their roles. A server is: `[host, port, role]`, where `role` is either `primary` or `replica`. | `["127.0.0.1", 5432, "primary"]` | diff --git a/src/client.rs b/src/client.rs index 3b0b0ea71..4e8556bb2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -9,11 +9,11 @@ use tokio::sync::broadcast::Receiver; use tokio::sync::mpsc::Sender; use crate::admin::{generate_server_info_for_admin, handle_admin}; -use crate::config::{get_config, Address}; +use crate::config::{get_config, Address, PoolMode}; use crate::constants::*; use crate::errors::Error; use crate::messages::*; -use crate::pool::{get_pool, ClientServerMap, ConnectionPool, PoolMode}; +use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; use crate::stats::{get_reporter, Reporter}; diff --git a/src/config.rs b/src/config.rs index e4c25a241..893f5b70d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -127,6 +127,7 @@ pub struct User { pub username: String, pub password: String, pub pool_size: u32, + #[serde(default)] // 0 pub statement_timeout: u64, } @@ -144,34 +145,81 @@ impl Default for User { /// General configuration. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct General { + #[serde(default = "General::default_host")] pub host: String, + + #[serde(default = "General::default_port")] pub port: i16, + pub enable_prometheus_exporter: Option, pub prometheus_exporter_port: i16, + + #[serde(default = "General::default_connect_timeout")] pub connect_timeout: u64, - pub healthcheck_timeout: u64, + + #[serde(default = "General::default_shutdown_timeout")] pub shutdown_timeout: u64, + + #[serde(default = "General::default_healthcheck_timeout")] + pub healthcheck_timeout: u64, + + #[serde(default = "General::default_healthcheck_delay")] pub healthcheck_delay: u64, + + #[serde(default = "General::default_ban_time")] pub ban_time: i64, + + #[serde(default)] // False pub autoreload: bool, + pub tls_certificate: Option, pub tls_private_key: Option, pub admin_username: String, pub admin_password: String, } +impl General { + fn default_host() -> String { + "0.0.0.0".into() + } + + fn default_port() -> i16 { + 5432 + } + + fn default_connect_timeout() -> u64 { + 1000 + } + + fn default_shutdown_timeout() -> u64 { + 60000 + } + + fn default_healthcheck_timeout() -> u64 { + 1000 + } + + fn default_healthcheck_delay() -> u64 { + 30000 + } + + fn default_ban_time() -> i64 { + 60 + } +} + impl Default for General { fn default() -> General { General { - host: String::from("localhost"), - port: 5432, + host: General::default_host(), + port: General::default_port(), enable_prometheus_exporter: Some(false), prometheus_exporter_port: 9930, - connect_timeout: 5000, - healthcheck_timeout: 1000, - shutdown_timeout: 60000, - healthcheck_delay: 30000, - ban_time: 60, + connect_timeout: General::default_connect_timeout(), + shutdown_timeout: General::default_shutdown_timeout(), + healthcheck_timeout: General::default_healthcheck_timeout(), + healthcheck_delay: General::default_healthcheck_delay(), + ban_time: General::default_ban_time(), autoreload: false, tls_certificate: None, tls_private_key: None, @@ -180,25 +228,61 @@ impl Default for General { } } } + +/// Pool mode: +/// - transaction: server serves one transaction, +/// - session: server is attached to the client. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Copy)] +pub enum PoolMode { + #[serde(alias = "transaction", alias = "Transaction")] + Transaction, + + #[serde(alias = "session", alias = "Session")] + Session, +} + +impl ToString for PoolMode { + fn to_string(&self) -> String { + match *self { + PoolMode::Transaction => "transaction".to_string(), + PoolMode::Session => "session".to_string(), + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Pool { - pub pool_mode: String, + #[serde(default = "Pool::default_pool_mode")] + pub pool_mode: PoolMode, + pub default_role: String, + + #[serde(default)] // False pub query_parser_enabled: bool, + + #[serde(default)] // False pub primary_reads_enabled: bool, + pub sharding_function: String, pub shards: HashMap, pub users: HashMap, } + +impl Pool { + fn default_pool_mode() -> PoolMode { + PoolMode::Transaction + } +} + impl Default for Pool { fn default() -> Pool { Pool { - pool_mode: String::from("transaction"), + pool_mode: Pool::default_pool_mode(), shards: HashMap::from([(String::from("1"), Shard::default())]), users: HashMap::default(), default_role: String::from("any"), query_parser_enabled: false, - primary_reads_enabled: true, + primary_reads_enabled: false, sharding_function: "pg_bigint_hash".to_string(), } } @@ -231,10 +315,6 @@ impl Default for Shard { } } -fn default_path() -> String { - String::from("pgcat.toml") -} - /// Configuration wrapper. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Config { @@ -249,17 +329,23 @@ pub struct Config { // [main.subconf] // field1_under_subconf = 1 // field3_under_main = 3 # This field will be interpreted as being under subconf and not under main - #[serde(default = "default_path")] + #[serde(default = "Config::default_path")] pub path: String, pub general: General, pub pools: HashMap, } +impl Config { + fn default_path() -> String { + String::from("pgcat.toml") + } +} + impl Default for Config { fn default() -> Config { Config { - path: String::from("pgcat.toml"), + path: Config::default_path(), general: General::default(), pools: HashMap::default(), } @@ -275,7 +361,7 @@ impl From<&Config> for std::collections::HashMap { [ ( format!("pools.{}.pool_mode", pool_name), - pool.pool_mode.clone(), + pool.pool_mode.to_string(), ), ( format!("pools.{}.primary_reads_enabled", pool_name), @@ -383,7 +469,10 @@ impl Config { .sum::() .to_string() ); - info!("[pool: {}] Pool mode: {}", pool_name, pool_config.pool_mode); + info!( + "[pool: {}] Pool mode: {:?}", + pool_name, pool_config.pool_mode + ); info!( "[pool: {}] Sharding function: {}", pool_name, pool_config.sharding_function @@ -513,18 +602,6 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; - match pool.pool_mode.as_ref() { - "transaction" => (), - "session" => (), - other => { - error!( - "pool_mode can be 'session' or 'transaction', got: '{}'", - other - ); - return Err(Error::BadConfig); - } - }; - for shard in &pool.shards { // We use addresses as unique identifiers, // let's make sure they are unique in the config as well. diff --git a/src/pool.rs b/src/pool.rs index f81a1e075..af64fd386 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -12,7 +12,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; -use crate::config::{get_config, Address, Role, User}; +use crate::config::{get_config, Address, PoolMode, Role, User}; use crate::errors::Error; use crate::server::Server; @@ -27,24 +27,6 @@ pub type PoolMap = HashMap<(String, String), ConnectionPool>; /// The pool is recreated dynamically when the config is reloaded. pub static POOLS: Lazy> = Lazy::new(|| ArcSwap::from_pointee(HashMap::default())); -/// Pool mode: -/// - transaction: server serves one transaction, -/// - session: server is attached to the client. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum PoolMode { - Session, - Transaction, -} - -impl std::fmt::Display for PoolMode { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match *self { - PoolMode::Session => write!(f, "session"), - PoolMode::Transaction => write!(f, "transaction"), - } - } -} - /// Pool settings. #[derive(Clone, Debug)] pub struct PoolSettings { @@ -199,11 +181,7 @@ impl ConnectionPool { stats: get_reporter(), server_info: BytesMut::new(), settings: PoolSettings { - pool_mode: match pool_config.pool_mode.as_str() { - "transaction" => PoolMode::Transaction, - "session" => PoolMode::Session, - _ => unreachable!(), - }, + pool_mode: pool_config.pool_mode, // shards: pool_config.shards.clone(), shards: shard_ids.len(), user: user.clone(), diff --git a/src/query_router.rs b/src/query_router.rs index 85d4f8b4e..34745edd2 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -359,8 +359,8 @@ impl QueryRouter { #[cfg(test)] mod test { use super::*; + use crate::config::PoolMode; use crate::messages::simple_query; - use crate::pool::PoolMode; use crate::sharding::ShardingFunction; use bytes::BufMut; From 1ef8b7a8a71895b7b0894bf6b31b158a0ff342b9 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 23 Sep 2022 11:08:38 -0500 Subject: [PATCH 214/551] Log failed client logins (#173) * Log failed client logins * more logging * remove clones * remove --- src/client.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/client.rs b/src/client.rs index 4e8556bb2..cdef30f6e 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,6 +1,6 @@ /// Handle clients by pretending to be a PostgreSQL server. use bytes::{Buf, BufMut, BytesMut}; -use log::{debug, error, info, trace}; +use log::{debug, error, info, trace, warn}; use std::collections::HashMap; use std::time::Instant; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; @@ -436,7 +436,7 @@ where ); if password_hash != password_response { - debug!("Password authentication failed"); + warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name); wrong_password(&mut write, username).await?; return Err(Error::ClientError); @@ -458,6 +458,7 @@ where ) .await?; + warn!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name); return Err(Error::ClientError); } }; @@ -466,7 +467,7 @@ where let password_hash = md5_hash_password(&username, &pool.settings.user.password, &salt); if password_hash != password_response { - debug!("Password authentication failed"); + warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name); wrong_password(&mut write, username).await?; return Err(Error::ClientError); @@ -658,6 +659,8 @@ where ), ) .await?; + + warn!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", self.pool_name, self.username, self.application_name); return Err(Error::ClientError); } }; From a39bb7eab9df69b6198981815a63124c1440fcca Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 23 Sep 2022 11:32:05 -0700 Subject: [PATCH 215/551] Don't drop connections if DB hasn't changed (#175) * Don't drop connections if DB hasn't changed * Incoporate connect_timeout into the pool config * use the field --- src/config.rs | 42 +++++++++++++++++++++++++++++++++++------- src/pool.rs | 21 +++++++++++++++++++-- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/config.rs b/src/config.rs index 893f5b70d..e5c366c94 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,7 +4,7 @@ use log::{error, info}; use once_cell::sync::Lazy; use serde_derive::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; -use std::hash::Hash; +use std::hash::{Hash, Hasher}; use std::path::Path; use std::sync::Arc; use tokio::fs::File; @@ -122,7 +122,7 @@ impl Address { } /// PostgreSQL user. -#[derive(Clone, PartialEq, Hash, std::cmp::Eq, Serialize, Deserialize, Debug)] +#[derive(Clone, PartialEq, Hash, Eq, Serialize, Deserialize, Debug)] pub struct User { pub username: String, pub password: String, @@ -232,7 +232,7 @@ impl Default for General { /// Pool mode: /// - transaction: server serves one transaction, /// - session: server is attached to the client. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Copy)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Copy, Hash)] pub enum PoolMode { #[serde(alias = "transaction", alias = "Transaction")] Transaction, @@ -250,7 +250,7 @@ impl ToString for PoolMode { } } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct Pool { #[serde(default = "Pool::default_pool_mode")] pub pool_mode: PoolMode, @@ -263,11 +263,35 @@ pub struct Pool { #[serde(default)] // False pub primary_reads_enabled: bool, + #[serde(default = "General::default_connect_timeout")] + pub connect_timeout: u64, + pub sharding_function: String, pub shards: HashMap, pub users: HashMap, } +impl Hash for Pool { + fn hash(&self, state: &mut H) { + self.pool_mode.hash(state); + self.default_role.hash(state); + self.query_parser_enabled.hash(state); + self.primary_reads_enabled.hash(state); + self.sharding_function.hash(state); + self.connect_timeout.hash(state); + + for (key, value) in &self.shards { + key.hash(state); + value.hash(state); + } + + for (key, value) in &self.users { + key.hash(state); + value.hash(state); + } + } +} + impl Pool { fn default_pool_mode() -> PoolMode { PoolMode::Transaction @@ -284,6 +308,7 @@ impl Default for Pool { query_parser_enabled: false, primary_reads_enabled: false, sharding_function: "pg_bigint_hash".to_string(), + connect_timeout: General::default_connect_timeout(), } } } @@ -296,7 +321,7 @@ pub struct ServerConfig { } /// Shard configuration. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Hash, Eq)] pub struct Shard { pub database: String, pub servers: Vec, @@ -575,7 +600,10 @@ pub async fn parse(path: &str) -> Result<(), Error> { None => (), }; - for (pool_name, pool) in &config.pools { + for (pool_name, mut pool) in &mut config.pools { + // Copy the connect timeout over for hashing. + pool.connect_timeout = config.general.connect_timeout; + match pool.sharding_function.as_ref() { "pg_bigint_hash" => (), "sha1" => (), @@ -666,7 +694,7 @@ pub async fn reload_config(client_server_map: ClientServerMap) -> Result; /// This is atomic and safe and read-optimized. /// The pool is recreated dynamically when the config is reloaded. pub static POOLS: Lazy> = Lazy::new(|| ArcSwap::from_pointee(HashMap::default())); +static POOLS_HASH: Lazy>> = + Lazy::new(|| ArcSwap::from_pointee(HashSet::default())); /// Pool settings. #[derive(Clone, Debug)] @@ -101,9 +103,23 @@ impl ConnectionPool { let mut new_pools = HashMap::new(); let mut address_id = 0; + let mut pools_hash = (*(*POOLS_HASH.load())).clone(); + for (pool_name, pool_config) in &config.pools { + let changed = pools_hash.insert(pool_config.clone()); + + if !changed { + info!("[db: {}] has not changed", pool_name); + continue; + } + // There is one pool per database/user pair. for (_, user) in &pool_config.users { + info!( + "[pool: {}][user: {}] creating new pool", + pool_name, user.username + ); + let mut shards = Vec::new(); let mut addresses = Vec::new(); let mut banlist = Vec::new(); @@ -156,7 +172,7 @@ impl ConnectionPool { let pool = Pool::builder() .max_size(user.pool_size) .connection_timeout(std::time::Duration::from_millis( - config.general.connect_timeout, + pool_config.connect_timeout, )) .test_on_check_out(false) .build(manager) @@ -217,6 +233,7 @@ impl ConnectionPool { } POOLS.store(Arc::new(new_pools.clone())); + POOLS_HASH.store(Arc::new(pools_hash.clone())); Ok(()) } From 421c2100c5afbda810b40673297945f25198cebf Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 23 Sep 2022 12:06:07 -0700 Subject: [PATCH 216/551] Fix the pool fix (#176) * Always listen to the compiler * Its fine --- src/config.rs | 35 +++++++---------------------------- src/pool.rs | 22 +++++++++++++++++----- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/config.rs b/src/config.rs index e5c366c94..d13bee62d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,8 +3,8 @@ use arc_swap::ArcSwap; use log::{error, info}; use once_cell::sync::Lazy; use serde_derive::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::hash::{Hash, Hasher}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::hash::Hash; use std::path::Path; use std::sync::Arc; use tokio::fs::File; @@ -250,7 +250,7 @@ impl ToString for PoolMode { } } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] pub struct Pool { #[serde(default = "Pool::default_pool_mode")] pub pool_mode: PoolMode, @@ -267,29 +267,8 @@ pub struct Pool { pub connect_timeout: u64, pub sharding_function: String, - pub shards: HashMap, - pub users: HashMap, -} - -impl Hash for Pool { - fn hash(&self, state: &mut H) { - self.pool_mode.hash(state); - self.default_role.hash(state); - self.query_parser_enabled.hash(state); - self.primary_reads_enabled.hash(state); - self.sharding_function.hash(state); - self.connect_timeout.hash(state); - - for (key, value) in &self.shards { - key.hash(state); - value.hash(state); - } - - for (key, value) in &self.users { - key.hash(state); - value.hash(state); - } - } + pub shards: BTreeMap, + pub users: BTreeMap, } impl Pool { @@ -302,8 +281,8 @@ impl Default for Pool { fn default() -> Pool { Pool { pool_mode: Pool::default_pool_mode(), - shards: HashMap::from([(String::from("1"), Shard::default())]), - users: HashMap::default(), + shards: BTreeMap::from([(String::from("1"), Shard::default())]), + users: BTreeMap::default(), default_role: String::from("any"), query_parser_enabled: false, primary_reads_enabled: false, diff --git a/src/pool.rs b/src/pool.rs index bdc0174f6..d9c9e7d6d 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -108,13 +108,25 @@ impl ConnectionPool { for (pool_name, pool_config) in &config.pools { let changed = pools_hash.insert(pool_config.clone()); - if !changed { - info!("[db: {}] has not changed", pool_name); - continue; - } - // There is one pool per database/user pair. for (_, user) in &pool_config.users { + // If the pool hasn't changed, get existing reference and insert it into the new_pools. + // We replace all pools at the end, but if the reference is kept, the pool won't get re-created (bb8). + if !changed { + match get_pool(pool_name.clone(), user.username.clone()) { + Some(pool) => { + info!( + "[pool: {}][user: {}] has not changed", + pool_name, user.username + ); + new_pools + .insert((pool_name.clone(), user.username.clone()), pool.clone()); + continue; + } + None => (), + } + } + info!( "[pool: {}][user: {}] creating new pool", pool_name, user.username From daebcdb6a4f95e100b30fd54c61856fbc3bf8109 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 23 Sep 2022 12:24:30 -0700 Subject: [PATCH 217/551] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 1e77bafb2..3d7ee4b2c 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,7 @@ psql -h 127.0.0.1 -p 6432 -c 'SELECT 1' | **`user`** | | | | `name` | The user name. | `sharding_user` | | `password` | The user password in plaintext. | `hunter2` | -| `statement_timeout` | Timeout in milliseconds for how long a query takes to execute | `0 (disabled)` | - +| `statement_timeout` | Timeout in milliseconds for how long a query takes to execute | `0 (disabled)` | | | | | **`shards`** | Shards are numerically numbered starting from 0; the order in the config is preserved by the pooler to route queries accordingly. | `[shards.0]` | | `servers` | List of servers to connect to and their roles. A server is: `[host, port, role]`, where `role` is either `primary` or `replica`. | `["127.0.0.1", 5432, "primary"]` | From fb712585cd5da619832934aba99f3ca57cf8c78d Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sat, 24 Sep 2022 11:09:15 -0500 Subject: [PATCH 218/551] Set client state to idle after error (#179) * Set client state to idle after error * fmt * spelling * clean up --- src/stats.rs | 10 ++++++++-- tests/ruby/admin_spec.rb | 30 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/stats.rs b/src/stats.rs index 90b55ddd8..e37c88c7a 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -740,7 +740,10 @@ impl Collector { address_id, } => { match client_states.get_mut(&client_id) { - Some(client_info) => client_info.error_count += stat.value as u64, + Some(client_info) => { + client_info.state = ClientState::Idle; + client_info.error_count += stat.value as u64; + } None => warn!("Got event {:?} for unregistered client", stat.name), } @@ -757,7 +760,10 @@ impl Collector { address_id, } => { match client_states.get_mut(&client_id) { - Some(client_info) => client_info.error_count += stat.value as u64, + Some(client_info) => { + client_info.state = ClientState::Idle; + client_info.error_count += stat.value as u64; + } None => warn!("Got event {:?} for unregistered client", stat.name), } diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index a348146ef..3bc12641e 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -146,6 +146,36 @@ end end + context "client fail to checkout connection from the pool" do + it "counts clients as idle" do + new_configs = processes.pgcat.current_config + new_configs["general"]["connect_timeout"] = 500 + new_configs["general"]["ban_time"] = 1 + new_configs["general"]["shutdown_timeout"] = 1 + new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + + threads = [] + connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT pg_sleep(1)") rescue PG::SystemError } + end + + sleep(2) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_idle"]).to eq("5") + expect(results["sv_idle"]).to eq("1") + + threads.map(&:join) + connections.map(&:close) + end + end + context "clients overwhelm server pools" do let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) } From ba0199bfc9067dd6a0eaf01ab130f02e814191e1 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 28 Sep 2022 09:50:14 -0400 Subject: [PATCH 219/551] Change sharding config to enum and move validation of configs into public functions (#178) Moves config validation to own functions to enable tools to use them Moves sharding config to enum Makes defaults public Make connect_timeout on pool and option which is overwritten by general connect_timeout --- src/config.rs | 278 ++++++++++++++++++++++++++---------------------- src/pool.rs | 15 ++- src/sharding.rs | 14 ++- 3 files changed, 169 insertions(+), 138 deletions(-) diff --git a/src/config.rs b/src/config.rs index d13bee62d..1cb375950 100644 --- a/src/config.rs +++ b/src/config.rs @@ -13,6 +13,7 @@ use toml; use crate::errors::Error; use crate::pool::{ClientServerMap, ConnectionPool}; +use crate::sharding::ShardingFunction; use crate::tls::{load_certs, load_keys}; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -179,31 +180,31 @@ pub struct General { } impl General { - fn default_host() -> String { + pub fn default_host() -> String { "0.0.0.0".into() } - fn default_port() -> i16 { + pub fn default_port() -> i16 { 5432 } - fn default_connect_timeout() -> u64 { + pub fn default_connect_timeout() -> u64 { 1000 } - fn default_shutdown_timeout() -> u64 { + pub fn default_shutdown_timeout() -> u64 { 60000 } - fn default_healthcheck_timeout() -> u64 { + pub fn default_healthcheck_timeout() -> u64 { 1000 } - fn default_healthcheck_delay() -> u64 { + pub fn default_healthcheck_delay() -> u64 { 30000 } - fn default_ban_time() -> i64 { + pub fn default_ban_time() -> i64 { 60 } } @@ -211,15 +212,15 @@ impl General { impl Default for General { fn default() -> General { General { - host: General::default_host(), - port: General::default_port(), + host: Self::default_host(), + port: Self::default_port(), enable_prometheus_exporter: Some(false), prometheus_exporter_port: 9930, connect_timeout: General::default_connect_timeout(), - shutdown_timeout: General::default_shutdown_timeout(), - healthcheck_timeout: General::default_healthcheck_timeout(), - healthcheck_delay: General::default_healthcheck_delay(), - ban_time: General::default_ban_time(), + shutdown_timeout: Self::default_shutdown_timeout(), + healthcheck_timeout: Self::default_healthcheck_timeout(), + healthcheck_delay: Self::default_healthcheck_delay(), + ban_time: Self::default_ban_time(), autoreload: false, tls_certificate: None, tls_private_key: None, @@ -263,31 +264,61 @@ pub struct Pool { #[serde(default)] // False pub primary_reads_enabled: bool, - #[serde(default = "General::default_connect_timeout")] - pub connect_timeout: u64, + pub connect_timeout: Option, - pub sharding_function: String, + pub sharding_function: ShardingFunction, pub shards: BTreeMap, pub users: BTreeMap, } impl Pool { - fn default_pool_mode() -> PoolMode { + pub fn default_pool_mode() -> PoolMode { PoolMode::Transaction } + + pub fn validate(&self) -> Result<(), Error> { + match self.default_role.as_ref() { + "any" => (), + "primary" => (), + "replica" => (), + other => { + error!( + "Query router default_role must be 'primary', 'replica', or 'any', got: '{}'", + other + ); + return Err(Error::BadConfig); + } + }; + + for (shard_idx, shard) in &self.shards { + match shard_idx.parse::() { + Ok(_) => (), + Err(_) => { + error!( + "Shard '{}' is not a valid number, shards must be numbered starting at 0", + shard_idx + ); + return Err(Error::BadConfig); + } + }; + shard.validate()?; + } + + Ok(()) + } } impl Default for Pool { fn default() -> Pool { Pool { - pool_mode: Pool::default_pool_mode(), + pool_mode: Self::default_pool_mode(), shards: BTreeMap::from([(String::from("1"), Shard::default())]), users: BTreeMap::default(), default_role: String::from("any"), query_parser_enabled: false, primary_reads_enabled: false, - sharding_function: "pg_bigint_hash".to_string(), - connect_timeout: General::default_connect_timeout(), + sharding_function: ShardingFunction::PgBigintHash, + connect_timeout: None, } } } @@ -306,6 +337,45 @@ pub struct Shard { pub servers: Vec, } +impl Shard { + pub fn validate(&self) -> Result<(), Error> { + // We use addresses as unique identifiers, + // let's make sure they are unique in the config as well. + let mut dup_check = HashSet::new(); + let mut primary_count = 0; + + if self.servers.len() == 0 { + error!("Shard {} has no servers configured", self.database); + return Err(Error::BadConfig); + } + + for server in &self.servers { + dup_check.insert(server); + + // Check that we define only zero or one primary. + match server.role { + Role::Primary => primary_count += 1, + _ => (), + }; + } + + if primary_count > 1 { + error!( + "Shard {} has more than on primary configured", + self.database + ); + return Err(Error::BadConfig); + } + + if dup_check.len() != self.servers.len() { + error!("Shard {} contains duplicate server configs", self.database); + return Err(Error::BadConfig); + } + + Ok(()) + } +} + impl Default for Shard { fn default() -> Shard { Shard { @@ -326,7 +396,7 @@ pub struct Config { // so we should always put simple fields before nested fields // in all serializable structs to avoid ValueAfterTable errors // These errors occur when the toml serializer is about to produce - // ambigous toml structure like the one below + // ambiguous toml structure like the one below // [main] // field1_under_main = 1 // field2_under_main = 2 @@ -341,7 +411,7 @@ pub struct Config { } impl Config { - fn default_path() -> String { + pub fn default_path() -> String { String::from("pgcat.toml") } } @@ -349,7 +419,7 @@ impl Config { impl Default for Config { fn default() -> Config { Config { - path: Config::default_path(), + path: Self::default_path(), general: General::default(), pools: HashMap::default(), } @@ -381,7 +451,7 @@ impl From<&Config> for std::collections::HashMap { ), ( format!("pools.{}.sharding_function", pool_name), - pool.sharding_function.clone(), + pool.sharding_function.to_string(), ), ( format!("pools.{:?}.shard_count", pool_name), @@ -477,9 +547,18 @@ impl Config { "[pool: {}] Pool mode: {:?}", pool_name, pool_config.pool_mode ); + let connect_timeout = match pool_config.connect_timeout { + Some(connect_timeout) => connect_timeout, + None => self.general.connect_timeout, + }; + info!( + "[pool: {}] Connection timeout: {}ms", + pool_name, connect_timeout + ); info!( "[pool: {}] Sharding function: {}", - pool_name, pool_config.sharding_function + pool_name, + pool_config.sharding_function.to_string() ); info!( "[pool: {}] Primary reads: {}", @@ -512,6 +591,50 @@ impl Config { } } } + + pub fn validate(&mut self) -> Result<(), Error> { + // Validate TLS! + match self.general.tls_certificate.clone() { + Some(tls_certificate) => { + match load_certs(&Path::new(&tls_certificate)) { + Ok(_) => { + // Cert is okay, but what about the private key? + match self.general.tls_private_key.clone() { + Some(tls_private_key) => { + match load_keys(&Path::new(&tls_private_key)) { + Ok(_) => (), + Err(err) => { + error!( + "tls_private_key is incorrectly configured: {:?}", + err + ); + return Err(Error::BadConfig); + } + } + } + + None => { + error!("tls_certificate is set, but the tls_private_key is not"); + return Err(Error::BadConfig); + } + }; + } + + Err(err) => { + error!("tls_certificate is incorrectly configured: {:?}", err); + return Err(Error::BadConfig); + } + } + } + None => (), + }; + + for (_, pool) in &mut self.pools { + pool.validate()?; + } + + Ok(()) + } } /// Get a read-only instance of the configuration @@ -548,110 +671,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; - // Validate TLS! - match config.general.tls_certificate.clone() { - Some(tls_certificate) => { - match load_certs(&Path::new(&tls_certificate)) { - Ok(_) => { - // Cert is okay, but what about the private key? - match config.general.tls_private_key.clone() { - Some(tls_private_key) => match load_keys(&Path::new(&tls_private_key)) { - Ok(_) => (), - Err(err) => { - error!("tls_private_key is incorrectly configured: {:?}", err); - return Err(Error::BadConfig); - } - }, - - None => { - error!("tls_certificate is set, but the tls_private_key is not"); - return Err(Error::BadConfig); - } - }; - } - - Err(err) => { - error!("tls_certificate is incorrectly configured: {:?}", err); - return Err(Error::BadConfig); - } - } - } - None => (), - }; - - for (pool_name, mut pool) in &mut config.pools { - // Copy the connect timeout over for hashing. - pool.connect_timeout = config.general.connect_timeout; - - match pool.sharding_function.as_ref() { - "pg_bigint_hash" => (), - "sha1" => (), - _ => { - error!( - "Supported sharding functions are: 'pg_bigint_hash', 'sha1', got: '{}' in pool {} settings", - pool.sharding_function, - pool_name - ); - return Err(Error::BadConfig); - } - }; - - match pool.default_role.as_ref() { - "any" => (), - "primary" => (), - "replica" => (), - other => { - error!( - "Query router default_role must be 'primary', 'replica', or 'any', got: '{}'", - other - ); - return Err(Error::BadConfig); - } - }; - - for shard in &pool.shards { - // We use addresses as unique identifiers, - // let's make sure they are unique in the config as well. - let mut dup_check = HashSet::new(); - let mut primary_count = 0; - - match shard.0.parse::() { - Ok(_) => (), - Err(_) => { - error!( - "Shard '{}' is not a valid number, shards must be numbered starting at 0", - shard.0 - ); - return Err(Error::BadConfig); - } - }; - - if shard.1.servers.len() == 0 { - error!("Shard {} has no servers configured", shard.0); - return Err(Error::BadConfig); - } - - for server in &shard.1.servers { - dup_check.insert(server); - - // Check that we define only zero or one primary. - match server.role { - Role::Primary => primary_count += 1, - _ => (), - }; - } - - if primary_count > 1 { - error!("Shard {} has more than on primary configured", &shard.0); - return Err(Error::BadConfig); - } - - if dup_check.len() != shard.1.servers.len() { - error!("Shard {} contains duplicate server configs", &shard.0); - return Err(Error::BadConfig); - } - } - } + config.validate()?; config.path = path.to_string(); diff --git a/src/pool.rs b/src/pool.rs index d9c9e7d6d..815a2b8b2 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -181,11 +181,14 @@ impl ConnectionPool { get_reporter(), ); + let connect_timeout = match pool_config.connect_timeout { + Some(connect_timeout) => connect_timeout, + None => config.general.connect_timeout, + }; + let pool = Pool::builder() .max_size(user.pool_size) - .connection_timeout(std::time::Duration::from_millis( - pool_config.connect_timeout, - )) + .connection_timeout(std::time::Duration::from_millis(connect_timeout)) .test_on_check_out(false) .build(manager) .await @@ -221,11 +224,7 @@ impl ConnectionPool { }, query_parser_enabled: pool_config.query_parser_enabled.clone(), primary_reads_enabled: pool_config.primary_reads_enabled, - sharding_function: match pool_config.sharding_function.as_str() { - "pg_bigint_hash" => ShardingFunction::PgBigintHash, - "sha1" => ShardingFunction::Sha1, - _ => unreachable!(), - }, + sharding_function: pool_config.sharding_function, }, }; diff --git a/src/sharding.rs b/src/sharding.rs index c332c6012..c5ab45e70 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -1,3 +1,4 @@ +use serde_derive::{Deserialize, Serialize}; /// Implements various sharding functions. use sha1::{Digest, Sha1}; @@ -5,12 +6,23 @@ use sha1::{Digest, Sha1}; const PARTITION_HASH_SEED: u64 = 0x7A5B22367996DCFD; /// The sharding functions we support. -#[derive(Debug, PartialEq, Copy, Clone)] +#[derive(Debug, PartialEq, Copy, Clone, Serialize, Deserialize, Hash, std::cmp::Eq)] pub enum ShardingFunction { + #[serde(alias = "pg_bigint_hash", alias = "PgBigintHash")] PgBigintHash, + #[serde(alias = "sha1", alias = "Sha1")] Sha1, } +impl ToString for ShardingFunction { + fn to_string(&self) -> String { + match *self { + ShardingFunction::PgBigintHash => "pg_bigint_hash".to_string(), + ShardingFunction::Sha1 => "sha1".to_string(), + } + } +} + /// The sharder. pub struct Sharder { /// Number of shards in the cluster. From a387a03eba28354ad8126159357a9cd60a8ab854 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 1 Oct 2022 11:25:59 -0600 Subject: [PATCH 220/551] Replace a few types with more developer-friendly names (#182) * Replace a few types with more developer-friendly names * UserPool -> PoolIdentifier --- src/admin.rs | 12 ++++++------ src/client.rs | 4 ++-- src/pool.rs | 47 ++++++++++++++++++++++++++++++++++++++--------- src/stats.rs | 6 +++--- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 1aa2bced1..d794b86ae 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -215,16 +215,16 @@ where let mut res = BytesMut::new(); res.put(row_description(&columns)); - for ((pool_name, username), pool) in get_all_pools() { + for (user_pool, pool) in get_all_pools() { let def = HashMap::default(); let pool_stats = all_pool_stats - .get(&(pool_name.clone(), username.clone())) + .get(&(user_pool.db.clone(), user_pool.user.clone())) .unwrap_or(&def); let pool_config = &pool.settings; let mut row = vec![ - pool_name.clone(), - username.clone(), + user_pool.db.clone(), + user_pool.user.clone(), pool_config.pool_mode.to_string(), ]; for column in &columns[3..columns.len()] { @@ -420,7 +420,7 @@ where let mut res = BytesMut::new(); res.put(row_description(&columns)); - for ((db, username), pool) in get_all_pools() { + for (user_pool, pool) in get_all_pools() { for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { let address = pool.address(shard, server); @@ -429,7 +429,7 @@ where None => HashMap::new(), }; - let mut row = vec![address.name(), db.clone(), username.clone()]; + let mut row = vec![address.name(), user_pool.db.clone(), user_pool.user.clone()]; for column in &columns[3..] { row.push(stats.get(column.0).unwrap_or(&0).to_string()); } diff --git a/src/client.rs b/src/client.rs index cdef30f6e..64dfa8ac8 100644 --- a/src/client.rs +++ b/src/client.rs @@ -446,7 +446,7 @@ where } // Authenticate normal user. else { - let pool = match get_pool(pool_name.clone(), username.clone()) { + let pool = match get_pool(&pool_name, &username) { Some(pool) => pool, None => { error_response( @@ -648,7 +648,7 @@ where // Get a pool instance referenced by the most up-to-date // pointer. This ensures we always read the latest config // when starting a query. - let pool = match get_pool(self.pool_name.clone(), self.username.clone()) { + let pool = match get_pool(&self.pool_name, &self.username) { Some(pool) => pool, None => { error_response( diff --git a/src/pool.rs b/src/pool.rs index 815a2b8b2..bb4525375 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -19,9 +19,15 @@ use crate::server::Server; use crate::sharding::ShardingFunction; use crate::stats::{get_reporter, Reporter}; +pub type ProcessId = i32; +pub type SecretKey = i32; +pub type ServerHost = String; +pub type ServerPort = u16; + pub type BanList = Arc>>>; -pub type ClientServerMap = Arc>>; -pub type PoolMap = HashMap<(String, String), ConnectionPool>; +pub type ClientServerMap = + Arc>>; +pub type PoolMap = HashMap; /// The connection pool, globally available. /// This is atomic and safe and read-optimized. /// The pool is recreated dynamically when the config is reloaded. @@ -29,6 +35,27 @@ pub static POOLS: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Ha static POOLS_HASH: Lazy>> = Lazy::new(|| ArcSwap::from_pointee(HashSet::default())); +/// An identifier for a PgCat pool, +/// a database visible to clients. +#[derive(Hash, Debug, Clone, PartialEq, Eq)] +pub struct PoolIdentifier { + // The name of the database clients want to connect to. + pub db: String, + + /// The username the client connects with. Each user gets its own pool. + pub user: String, +} + +impl PoolIdentifier { + /// Create a new user/pool identifier. + pub fn new(db: &str, user: &str) -> PoolIdentifier { + PoolIdentifier { + db: db.to_string(), + user: user.to_string(), + } + } +} + /// Pool settings. #[derive(Clone, Debug)] pub struct PoolSettings { @@ -113,14 +140,16 @@ impl ConnectionPool { // If the pool hasn't changed, get existing reference and insert it into the new_pools. // We replace all pools at the end, but if the reference is kept, the pool won't get re-created (bb8). if !changed { - match get_pool(pool_name.clone(), user.username.clone()) { + match get_pool(&pool_name, &user.username) { Some(pool) => { info!( "[pool: {}][user: {}] has not changed", pool_name, user.username ); - new_pools - .insert((pool_name.clone(), user.username.clone()), pool.clone()); + new_pools.insert( + PoolIdentifier::new(&pool_name, &user.username), + pool.clone(), + ); continue; } None => (), @@ -239,7 +268,7 @@ impl ConnectionPool { }; // There is one pool per database/user pair. - new_pools.insert((pool_name.clone(), user.username.clone()), pool); + new_pools.insert(PoolIdentifier::new(&pool_name, &user.username), pool); } } @@ -603,15 +632,15 @@ impl ManageConnection for ServerPool { } /// Get the connection pool -pub fn get_pool(db: String, user: String) -> Option { - match get_all_pools().get(&(db, user)) { +pub fn get_pool(db: &str, user: &str) -> Option { + match get_all_pools().get(&PoolIdentifier::new(&db, &user)) { Some(pool) => Some(pool.clone()), None => None, } } /// Get a pointer to all configured pools. -pub fn get_all_pools() -> HashMap<(String, String), ConnectionPool> { +pub fn get_all_pools() -> HashMap { return (*(*POOLS.load())).clone(); } diff --git a/src/stats.rs b/src/stats.rs index e37c88c7a..7998e4545 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -521,11 +521,11 @@ impl Collector { tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD / 15)); loop { interval.tick().await; - for ((pool_name, username), _pool) in get_all_pools() { + for (user_pool, _) in get_all_pools() { let _ = tx.try_send(Event { name: EventName::UpdateStats { - pool_name, - username, + pool_name: user_pool.db, + username: user_pool.user, }, value: 0, }); From daf6faff64d4543272ddc7cb695761fb55db8c98 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 5 Oct 2022 21:41:09 -0500 Subject: [PATCH 221/551] Fix maxwait metric (#183) Max wait was being reported as 0 after #159 This PR fixes that and adds test --- src/admin.rs | 8 +++++++- src/pool.rs | 4 ++-- src/stats.rs | 44 ++++++++++++++++++++++------------------ tests/ruby/admin_spec.rb | 22 ++++++++++++++++++++ 4 files changed, 55 insertions(+), 23 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index d794b86ae..42af315ed 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -228,7 +228,13 @@ where pool_config.pool_mode.to_string(), ]; for column in &columns[3..columns.len()] { - let value = pool_stats.get(column.0).unwrap_or(&0).to_string(); + let value = match column.0 { + "maxwait" => (pool_stats.get("maxwait_us").unwrap_or(&0) / 1_000_000).to_string(), + "maxwait_us" => { + (pool_stats.get("maxwait_us").unwrap_or(&0) % 1_000_000).to_string() + } + _other_values => pool_stats.get(column.0).unwrap_or(&0).to_string(), + }; row.push(value); } res.put(data_row(&row)); diff --git a/src/pool.rs b/src/pool.rs index bb4525375..edc549d85 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -333,7 +333,6 @@ impl ConnectionPool { role: Option, // primary or replica process_id: i32, // client id ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { - let now = Instant::now(); let mut candidates: Vec<&Address> = self.addresses[shard] .iter() .filter(|address| address.role == role) @@ -358,6 +357,7 @@ impl ConnectionPool { } // Indicate we're waiting on a server connection from a pool. + let now = Instant::now(); self.stats.client_waiting(process_id); // Check if we can connect @@ -397,7 +397,7 @@ impl ConnectionPool { match tokio::time::timeout( tokio::time::Duration::from_millis(healthcheck_timeout), - server.query(";"), // Cheap query (query parser not used in PG) + server.query(";"), // Cheap query as it skips the query planner ) .await { diff --git a/src/stats.rs b/src/stats.rs index 7998e4545..f6f812aa5 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -580,15 +580,15 @@ impl Collector { server_info.query_count += stat.value as u64; server_info.application_name = app_name; - let pool_stats = address_stat_lookup + let address_stats = address_stat_lookup .entry(server_info.address_id) .or_insert(HashMap::default()); - let counter = pool_stats + let counter = address_stats .entry("total_query_count".to_string()) .or_insert(0); *counter += stat.value; - let duration = pool_stats + let duration = address_stats .entry("total_query_time".to_string()) .or_insert(0); *duration += duration_ms as i64; @@ -681,26 +681,26 @@ impl Collector { Some(server_info) => { server_info.application_name = app_name; - let pool_stats = address_stat_lookup + let address_stats = address_stat_lookup .entry(server_info.address_id) .or_insert(HashMap::default()); - let counter = - pool_stats.entry("total_wait_time".to_string()).or_insert(0); + let counter = address_stats + .entry("total_wait_time".to_string()) + .or_insert(0); *counter += stat.value; - let counter = pool_stats.entry("maxwait_us".to_string()).or_insert(0); - let mic_part = stat.value % 1_000_000; - - // Report max time here - if mic_part > *counter { - *counter = mic_part; - } - - let counter = pool_stats.entry("maxwait".to_string()).or_insert(0); - let seconds = *counter / 1_000_000; + let pool_stats = pool_stat_lookup + .entry(( + server_info.pool_name.clone(), + server_info.username.clone(), + )) + .or_insert(HashMap::default()); - if seconds > *counter { - *counter = seconds; + // We record max wait in microseconds, we do the pgbouncer second/microsecond split on admin + let old_microseconds = + pool_stats.entry("maxwait_us".to_string()).or_insert(0); + if stat.value > *old_microseconds { + *old_microseconds = stat.value; } } None => (), @@ -903,8 +903,6 @@ impl Collector { "sv_active", "sv_tested", "sv_login", - "maxwait", - "maxwait_us", ] { pool_stats.insert(stat.to_string(), 0); } @@ -962,6 +960,12 @@ impl Collector { LATEST_CLIENT_STATS.store(Arc::new(client_states.clone())); LATEST_SERVER_STATS.store(Arc::new(server_states.clone())); LATEST_POOL_STATS.store(Arc::new(pool_stat_lookup.clone())); + + // Clear maxwait after reporting + pool_stat_lookup + .entry((pool_name.clone(), username.clone())) + .or_insert(HashMap::default()) + .insert("maxwait_us".to_string(), 0); } EventName::UpdateAverages { address_id } => { diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index 3bc12641e..40e7e1c2d 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -208,6 +208,28 @@ threads.map(&:join) connections.map(&:close) end + + it "show correct max_wait" do + threads = [] + connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") } + end + + sleep(2.5) # Allow time for stats to update + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + + expect(results["maxwait"]).to eq("1") + expect(results["maxwait_us"].to_i).to be_within(100_000).of(500_000) + + sleep(4.5) # Allow time for stats to update + results = admin_conn.async_exec("SHOW POOLS")[0] + expect(results["maxwait"]).to eq("0") + + threads.map(&:join) + connections.map(&:close) + end end end From b304ebede5f100b7f5565eedd98927603285ff3d Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sat, 8 Oct 2022 10:16:04 -0500 Subject: [PATCH 222/551] A bit faster get_pool (#187) * A bit faster get_pool * fmt --- src/pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pool.rs b/src/pool.rs index edc549d85..24505fa7b 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -633,7 +633,7 @@ impl ManageConnection for ServerPool { /// Get the connection pool pub fn get_pool(db: &str, user: &str) -> Option { - match get_all_pools().get(&PoolIdentifier::new(&db, &user)) { + match (*(*POOLS.load())).get(&PoolIdentifier::new(db, user)) { Some(pool) => Some(pool.clone()), None => None, } From 0947ac528b7c3b80979fb5aab74a35f6025e5384 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 13 Oct 2022 11:13:45 -0500 Subject: [PATCH 223/551] Use Jemalloc (#189) Jemalloc performs better than the standard allocator in various metrics (http://ithare.com/testing-memory-allocators-ptmalloc2-tcmalloc-hoard-jemalloc-while-trying-to-simulate-real-world-loads/). This PR makes changes to use Jemalloc as the global allocator for Pgcat. Windows is not officially supported by Pgcat but it should still compile but without Jemalloc as the allocator. --- Cargo.lock | 28 ++++++++++++++++++++++++++++ Cargo.toml | 3 +++ src/main.rs | 7 +++++++ 3 files changed, 38 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 2bced0610..746cf32d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,6 +171,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fs_extra" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" + [[package]] name = "futures-channel" version = "0.3.19" @@ -365,6 +371,27 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +[[package]] +name = "jemalloc-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" +dependencies = [ + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +dependencies = [ + "jemalloc-sys", + "libc", +] + [[package]] name = "js-sys" version = "0.3.58" @@ -547,6 +574,7 @@ dependencies = [ "exitcode", "hmac", "hyper", + "jemallocator", "log", "md-5", "num_cpus", diff --git a/Cargo.toml b/Cargo.toml index 48ba966d3..8284d2df7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,3 +34,6 @@ rustls-pemfile = "1" hyper = { version = "0.14", features = ["full"] } phf = { version = "0.11.1", features = ["macros"] } exitcode = "1.1.2" + +[target.'cfg(not(target_env = "msvc"))'.dependencies] +jemallocator = "0.3.2" diff --git a/src/main.rs b/src/main.rs index 467527c97..75aab69b9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,6 +37,13 @@ extern crate tokio; extern crate tokio_rustls; extern crate toml; +#[cfg(not(target_env = "msvc"))] +use jemallocator::Jemalloc; + +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + use log::{error, info, warn}; use parking_lot::Mutex; use pgcat::format_duration; From 579cf24b516af8572ae80781562fc7854a446478 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 13 Oct 2022 22:33:12 -0400 Subject: [PATCH 224/551] Don't send discard all when state is changed in transaction (#186) * Don't send discard all when state is changed in transaction * Remove unnecessary clone * spelling * Move transaction check to SET command * Add test for set command in transaction * type * Update comments * Update comments * use moves instead of clones for initial message * don't make message mutable * Update unwrap * but i'm not a wrapper * Add set local test * change continue --- src/client.rs | 57 ++++++++++++++++++++++------------------- src/server.rs | 14 ++++++++-- tests/ruby/misc_spec.rb | 25 ++++++++++++++++++ 3 files changed, 67 insertions(+), 29 deletions(-) diff --git a/src/client.rs b/src/client.rs index 64dfa8ac8..e72dbf794 100644 --- a/src/client.rs +++ b/src/client.rs @@ -601,7 +601,7 @@ where // in case the client is sending some custom protocol messages, e.g. // SET SHARDING KEY TO 'bigint'; - let mut message = tokio::select! { + let message = tokio::select! { _ = self.shutdown.recv() => { if !self.admin { error_response_terminal( @@ -792,6 +792,8 @@ where // Set application_name. server.set_name(&self.application_name).await?; + let mut initial_message = Some(message); + // Transaction loop. Multiple queries can be issued by the client here. // The connection belongs to the client until the transaction is over, // or until the client disconnects if we are in session mode. @@ -799,40 +801,42 @@ where // If the client is in session mode, no more custom protocol // commands will be accepted. loop { - let mut message = if message.len() == 0 { - trace!("Waiting for message inside transaction or in session mode"); + let message = match initial_message { + None => { + trace!("Waiting for message inside transaction or in session mode"); - match read_message(&mut self.read).await { - Ok(message) => message, - Err(err) => { - // Client disconnected inside a transaction. - // Clean up the server and re-use it. - server.checkin_cleanup().await?; + match read_message(&mut self.read).await { + Ok(message) => message, + Err(err) => { + // Client disconnected inside a transaction. + // Clean up the server and re-use it. + server.checkin_cleanup().await?; - return Err(err); + return Err(err); + } } } - } else { - let msg = message.clone(); - message.clear(); - msg + Some(message) => { + initial_message = None; + message + } }; // The message will be forwarded to the server intact. We still would like to // parse it below to figure out what to do with it. - let original = message.clone(); - let code = message.get_u8() as char; - let _len = message.get_i32() as usize; + // Safe to unwrap because we know this message has a certain length and has the code + // This reads the first byte without advancing the internal pointer and mutating the bytes + let code = *message.get(0).unwrap() as char; trace!("Message: {}", code); match code { - // ReadyForQuery + // Query 'Q' => { debug!("Sending query to server"); - self.send_and_receive_loop(code, original, server, &address, &pool) + self.send_and_receive_loop(code, message, server, &address, &pool) .await?; if !server.in_transaction() { @@ -858,25 +862,25 @@ where // Parse // The query with placeholders is here, e.g. `SELECT * FROM users WHERE email = $1 AND active = $2`. 'P' => { - self.buffer.put(&original[..]); + self.buffer.put(&message[..]); } // Bind // The placeholder's replacements are here, e.g. 'user@email.com' and 'true' 'B' => { - self.buffer.put(&original[..]); + self.buffer.put(&message[..]); } // Describe // Command a client can issue to describe a previously prepared named statement. 'D' => { - self.buffer.put(&original[..]); + self.buffer.put(&message[..]); } // Execute // Execute a prepared statement prepared in `P` and bound in `B`. 'E' => { - self.buffer.put(&original[..]); + self.buffer.put(&message[..]); } // Sync @@ -884,9 +888,8 @@ where 'S' => { debug!("Sending query to server"); - self.buffer.put(&original[..]); + self.buffer.put(&message[..]); - // Clone after freeze does not allocate let first_message_code = (*self.buffer.get(0).unwrap_or(&0)) as char; // Almost certainly true @@ -929,14 +932,14 @@ where 'd' => { // Forward the data to the server, // don't buffer it since it can be rather large. - self.send_server_message(server, original, &address, &pool) + self.send_server_message(server, message, &address, &pool) .await?; } // CopyDone or CopyFail // Copy is done, successfully or not. 'c' | 'f' => { - self.send_server_message(server, original, &address, &pool) + self.send_server_message(server, message, &address, &pool) .await?; let response = self.receive_server_message(server, &address, &pool).await?; diff --git a/src/server.rs b/src/server.rs index dbac9bc09..d191eb74d 100644 --- a/src/server.rs +++ b/src/server.rs @@ -457,7 +457,17 @@ impl Server { // which can leak between clients. This is a best effort to block bad clients // from poisoning a transaction-mode pool by setting inappropriate session variables match command_tag.as_str() { - "SET\0" | "PREPARE\0" => { + "SET\0" => { + // We don't detect set statements in transactions + // No great way to differentiate between set and set local + // As a result, we will miss cases when set statements are used in transactions + // This will reduce amount of discard statements sent + if !self.in_transaction { + debug!("Server connection marked for clean up"); + self.needs_cleanup = true; + } + } + "PREPARE\0" => { debug!("Server connection marked for clean up"); self.needs_cleanup = true; } @@ -595,7 +605,7 @@ impl Server { self.query("ROLLBACK").await?; } - // Client disconnected but it perfromed session-altering operations such as + // Client disconnected but it performed session-altering operations such as // SET statement_timeout to 1 or create a prepared statement. We clear that // to avoid leaking state between clients. For performance reasons we only // send `DISCARD ALL` if we think the session is altered instead of just sending diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index 6e79e1a4f..1f5bf4217 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -189,5 +189,30 @@ expect(processes.primary.count_query("DISCARD ALL")).to eq(10) end end + + context "transaction mode with transactions" do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5, "transaction") } + it "Does not clear set statement state when declared in a transaction" do + 10.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'primary'") + conn.async_exec("BEGIN") + conn.async_exec("SET statement_timeout to 1000") + conn.async_exec("COMMIT") + conn.close + end + expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + + 10.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'primary'") + conn.async_exec("BEGIN") + conn.async_exec("SET LOCAL statement_timeout to 1000") + conn.async_exec("COMMIT") + conn.close + end + expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + end + end end end From 554227fc17aaf6362a03af7698eff1b193f1208a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 23 Oct 2022 16:59:51 -0700 Subject: [PATCH 225/551] Re-enable query parser and parse multiple statements (#191) * Re-enable query parser and parse multiple statements * no diff --- Cargo.lock | 4 +- Cargo.toml | 2 +- src/query_router.rs | 112 +++++++++++++++++++++++++++++--------------- 3 files changed, 77 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 746cf32d2..6b2548325 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -875,9 +875,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.23.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beb13adabbdda01b63d595f38c8bfd19a361e697fd94ce0098a634077bc5b25" +checksum = "86be66ea0b2b22749cfa157d16e2e84bf793e626a3375f4d378dc289fa03affb" dependencies = [ "log", ] diff --git a/Cargo.toml b/Cargo.toml index 8284d2df7..e06b6930a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = "0.23.0" +sqlparser = "0.26.0" log = "0.4" arc-swap = "1" env_logger = "0.9" diff --git a/src/query_router.rs b/src/query_router.rs index 34745edd2..269645b2a 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -50,10 +50,10 @@ pub struct QueryRouter { active_role: Option, /// Should we try to parse queries to route them to replicas or primary automatically - query_parser_enabled: bool, + query_parser_enabled: Option, /// Include the primary into the replica pool for reads. - primary_reads_enabled: bool, + primary_reads_enabled: Option, /// Pool configuration. pool_settings: PoolSettings, @@ -95,8 +95,8 @@ impl QueryRouter { QueryRouter { active_shard: None, active_role: None, - query_parser_enabled: false, - primary_reads_enabled: false, + query_parser_enabled: None, + primary_reads_enabled: None, pool_settings: PoolSettings::default(), } } @@ -172,7 +172,7 @@ impl QueryRouter { Some(Role::Primary) => Role::Primary.to_string(), Some(Role::Replica) => Role::Replica.to_string(), None => { - if self.query_parser_enabled { + if self.query_parser_enabled() { String::from("auto") } else { String::from("any") @@ -180,7 +180,7 @@ impl QueryRouter { } }, - Command::ShowPrimaryReads => match self.primary_reads_enabled { + Command::ShowPrimaryReads => match self.primary_reads_enabled() { true => String::from("on"), false => String::from("off"), }, @@ -207,28 +207,28 @@ impl QueryRouter { Command::SetServerRole => { self.active_role = match value.to_ascii_lowercase().as_ref() { "primary" => { - self.query_parser_enabled = false; + self.query_parser_enabled = Some(false); Some(Role::Primary) } "replica" => { - self.query_parser_enabled = false; + self.query_parser_enabled = Some(false); Some(Role::Replica) } "any" => { - self.query_parser_enabled = false; + self.query_parser_enabled = Some(false); None } "auto" => { - self.query_parser_enabled = true; + self.query_parser_enabled = Some(true); None } "default" => { self.active_role = self.pool_settings.default_role; - self.query_parser_enabled = self.query_parser_enabled; + self.query_parser_enabled = None; self.active_role } @@ -239,13 +239,13 @@ impl QueryRouter { Command::SetPrimaryReads => { if value == "on" { debug!("Setting primary reads to on"); - self.primary_reads_enabled = true; + self.primary_reads_enabled = Some(true); } else if value == "off" { debug!("Setting primary reads to off"); - self.primary_reads_enabled = false; + self.primary_reads_enabled = Some(false); } else if value == "default" { debug!("Setting primary reads to default"); - self.primary_reads_enabled = self.pool_settings.primary_reads_enabled; + self.primary_reads_enabled = None; } } @@ -300,34 +300,44 @@ impl QueryRouter { let ast = match Parser::parse_sql(&PostgreSqlDialect {}, &query) { Ok(ast) => ast, Err(err) => { - debug!("{}", err.to_string()); + // SELECT ... FOR UPDATE won't get parsed correctly. + error!("{}: {}", err, query); + self.active_role = Some(Role::Primary); return false; } }; + debug!("AST: {:?}", ast); + if ast.len() == 0 { + // That's weird, no idea, let's go to primary + self.active_role = Some(Role::Primary); return false; } - match ast[0] { - // All transactions go to the primary, probably a write. - StartTransaction { .. } => { - self.active_role = Some(Role::Primary); - } + for q in &ast { + match q { + // All transactions go to the primary, probably a write. + StartTransaction { .. } => { + self.active_role = Some(Role::Primary); + break; + } - // Likely a read-only query - Query { .. } => { - self.active_role = match self.primary_reads_enabled { - false => Some(Role::Replica), // If primary should not be receiving reads, use a replica. - true => None, // Any server role is fine in this case. + // Likely a read-only query + Query { .. } => { + self.active_role = match self.primary_reads_enabled() { + false => Some(Role::Replica), // If primary should not be receiving reads, use a replica. + true => None, // Any server role is fine in this case. + } } - } - // Likely a write - _ => { - self.active_role = Some(Role::Primary); - } - }; + // Likely a write + _ => { + self.active_role = Some(Role::Primary); + break; + } + }; + } true } @@ -350,9 +360,18 @@ impl QueryRouter { } /// Should we attempt to parse queries? - #[allow(dead_code)] pub fn query_parser_enabled(&self) -> bool { - self.query_parser_enabled + match self.query_parser_enabled { + None => self.pool_settings.query_parser_enabled, + Some(value) => value, + } + } + + pub fn primary_reads_enabled(&self) -> bool { + match self.primary_reads_enabled { + None => self.pool_settings.primary_reads_enabled, + Some(value) => value, + } } } @@ -616,7 +635,7 @@ mod test { assert!(qr.query_parser_enabled()); let query = simple_query("SET SERVER ROLE TO 'default'"); assert!(qr.try_execute_command(query) != None); - assert!(qr.query_parser_enabled()); + assert!(!qr.query_parser_enabled()); } #[test] @@ -635,16 +654,16 @@ mod test { let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); assert_eq!(qr.active_shard, None); - assert_eq!(qr.query_parser_enabled, false); - assert_eq!(qr.primary_reads_enabled, false); + assert_eq!(qr.query_parser_enabled, None); + assert_eq!(qr.primary_reads_enabled, None); // Internal state must not be changed due to this, only defaults qr.update_pool_settings(pool_settings.clone()); assert_eq!(qr.active_role, None); assert_eq!(qr.active_shard, None); - assert_eq!(qr.query_parser_enabled, false); - assert_eq!(qr.primary_reads_enabled, false); + assert_eq!(qr.query_parser_enabled(), true); + assert_eq!(qr.primary_reads_enabled(), false); let q1 = simple_query("SET SERVER ROLE TO 'primary'"); assert!(qr.try_execute_command(q1) != None); @@ -654,4 +673,21 @@ mod test { assert!(qr.try_execute_command(q2) != None); assert_eq!(qr.active_role.unwrap(), pool_settings.clone().default_role); } + + #[test] + fn test_parse_multiple_queries() { + QueryRouter::setup(); + + let mut qr = QueryRouter::new(); + assert!(qr.infer_role(simple_query("BEGIN; SELECT 1; COMMIT;"))); + assert_eq!(qr.role(), Role::Primary); + + assert!(qr.infer_role(simple_query("SELECT 1; SELECT 2;"))); + assert_eq!(qr.role(), Role::Replica); + + assert!(qr.infer_role(simple_query( + "SELECT 123; INSERT INTO t VALUES (5); SELECT 1;" + ))); + assert_eq!(qr.role(), Role::Primary); + } } From bd3af6847707114ec447192f86e0a508f4904f35 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 24 Oct 2022 11:05:33 -0700 Subject: [PATCH 226/551] Fix docker-compose (#193) Fix docker-compose local build --- README.md | 2 +- examples/docker/pgcat.toml | 37 +++++-------------------------------- 2 files changed, 6 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 3d7ee4b2c..313c11008 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ For quick local example, use the Docker Compose environment provided: docker-compose up # In a new terminal: -psql -h 127.0.0.1 -p 6432 -c 'SELECT 1' +PGPASSWORD=postgres psql -h 127.0.0.1 -p 6432 -U postgres -c 'SELECT 1' ``` ### Config diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index 85a3f466d..97f1c9c35 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -48,7 +48,7 @@ admin_password = "postgres" # configs are structured as pool. # the pool_name is what clients use as database name when connecting # For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded" -[pools.sharded] +[pools.postgres] # Pool mode (see PgBouncer docs for more). # session: one server connection per connected client # transaction: one server connection per client transaction @@ -84,7 +84,7 @@ primary_reads_enabled = true sharding_function = "pg_bigint_hash" # Credentials for users that may connect to this cluster -[pools.sharded.users.0] +[pools.postgres.users.0] username = "postgres" password = "postgres" # Maximum number of server connections that can be established for this user @@ -95,14 +95,8 @@ pool_size = 9 # Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. statement_timeout = 0 -[pools.sharded.users.1] -username = "postgres" -password = "postgres" -pool_size = 21 -statement_timeout = 15000 - # Shard 0 -[pools.sharded.shards.0] +[pools.postgres.shards.0] # [ host, port, role ] servers = [ [ "postgres", 5432, "primary" ], @@ -111,37 +105,16 @@ servers = [ # Database name (e.g. "postgres") database = "postgres" -[pools.sharded.shards.1] +[pools.postgres.shards.1] servers = [ [ "postgres", 5432, "primary" ], [ "postgres", 5432, "replica" ], ] database = "postgres" -[pools.sharded.shards.2] +[pools.postgres.shards.2] servers = [ [ "postgres", 5432, "primary" ], [ "postgres", 5432, "replica" ], ] database = "postgres" - - -[pools.simple_db] -pool_mode = "session" -default_role = "primary" -query_parser_enabled = true -primary_reads_enabled = true -sharding_function = "pg_bigint_hash" - -[pools.simple_db.users.0] -username = "postgres" -password = "postgres" -pool_size = 5 -statement_timeout = 0 - -[pools.simple_db.shards.0] -servers = [ - [ "postgres", 5432, "primary" ], - [ "postgres", 5432, "replica" ] -] -database = "postgres" From a17720a8da8d63a7b4d9f27e16eaa20798c63c1a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 25 Oct 2022 11:47:41 -0700 Subject: [PATCH 227/551] Automatic sharding: part one of many (#194) Starting automatic sharding --- pgcat.toml | 3 + src/client.rs | 2 +- src/config.rs | 9 +++ src/pool.rs | 5 ++ src/query_router.rs | 167 +++++++++++++++++++++++++++++++++++++++----- 5 files changed, 169 insertions(+), 17 deletions(-) diff --git a/pgcat.toml b/pgcat.toml index 9125afd5c..8d588043d 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -83,6 +83,9 @@ primary_reads_enabled = true # sharding_function = "pg_bigint_hash" +# Automatically parse this from queries and route queries to the right shard! +automatic_sharding_key = "id" + # Credentials for users that may connect to this cluster [pools.sharded_db.users.0] username = "sharding_user" diff --git a/src/client.rs b/src/client.rs index e72dbf794..7db920525 100644 --- a/src/client.rs +++ b/src/client.rs @@ -672,7 +672,7 @@ where // Normal query, not a custom command. None => { if query_router.query_parser_enabled() { - query_router.infer_role(message.clone()); + query_router.infer(message.clone()); } } diff --git a/src/config.rs b/src/config.rs index 1cb375950..647b0166c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -267,6 +267,10 @@ pub struct Pool { pub connect_timeout: Option, pub sharding_function: ShardingFunction, + + #[serde(default = "Pool::default_automatic_sharding_key")] + pub automatic_sharding_key: Option, + pub shards: BTreeMap, pub users: BTreeMap, } @@ -276,6 +280,10 @@ impl Pool { PoolMode::Transaction } + pub fn default_automatic_sharding_key() -> Option { + None + } + pub fn validate(&self) -> Result<(), Error> { match self.default_role.as_ref() { "any" => (), @@ -318,6 +326,7 @@ impl Default for Pool { query_parser_enabled: false, primary_reads_enabled: false, sharding_function: ShardingFunction::PgBigintHash, + automatic_sharding_key: None, connect_timeout: None, } } diff --git a/src/pool.rs b/src/pool.rs index 24505fa7b..2b80f7bab 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -79,6 +79,9 @@ pub struct PoolSettings { // Sharding function. pub sharding_function: ShardingFunction, + + // Sharding key + pub automatic_sharding_key: Option, } impl Default for PoolSettings { @@ -91,6 +94,7 @@ impl Default for PoolSettings { query_parser_enabled: false, primary_reads_enabled: true, sharding_function: ShardingFunction::PgBigintHash, + automatic_sharding_key: None, } } } @@ -254,6 +258,7 @@ impl ConnectionPool { query_parser_enabled: pool_config.query_parser_enabled.clone(), primary_reads_enabled: pool_config.primary_reads_enabled, sharding_function: pool_config.sharding_function, + automatic_sharding_key: pool_config.automatic_sharding_key.clone(), }, }; diff --git a/src/query_router.rs b/src/query_router.rs index 269645b2a..f65a77342 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -5,6 +5,7 @@ use log::{debug, error}; use once_cell::sync::OnceCell; use regex::{Regex, RegexSet}; use sqlparser::ast::Statement::{Query, StartTransaction}; +use sqlparser::ast::{BinaryOperator, Expr, SetExpr, Value}; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; @@ -12,6 +13,8 @@ use crate::config::Role; use crate::pool::PoolSettings; use crate::sharding::Sharder; +use std::collections::BTreeSet; + /// Regexes used to parse custom commands. const CUSTOM_SQL_REGEXES: [&str; 7] = [ r"(?i)^ *SET SHARDING KEY TO '?([0-9]+)'? *;? *$", @@ -256,7 +259,7 @@ impl QueryRouter { } /// Try to infer which server to connect to based on the contents of the query. - pub fn infer_role(&mut self, mut buf: BytesMut) -> bool { + pub fn infer(&mut self, mut buf: BytesMut) -> bool { debug!("Inferring role"); let code = buf.get_u8() as char; @@ -324,7 +327,21 @@ impl QueryRouter { } // Likely a read-only query - Query { .. } => { + Query(query) => { + match &self.pool_settings.automatic_sharding_key { + Some(_) => { + // TODO: if we have multiple queries in the same message, + // we can either split them and execute them individually + // or discard shard selection. If they point to the same shard though, + // we can let them through as-is. + // This is basically building a database now :) + self.active_shard = self.infer_shard(query); + debug!("Automatically using shard: {:?}", self.active_shard); + } + + None => (), + }; + self.active_role = match self.primary_reads_enabled() { false => Some(Role::Replica), // If primary should not be receiving reads, use a replica. true => None, // Any server role is fine in this case. @@ -342,6 +359,118 @@ impl QueryRouter { true } + /// A `selection` is the `WHERE` clause. This parses + /// the clause and extracts the sharding key, if present. + fn selection_parser(&self, expr: &Expr) -> Vec { + let mut result = Vec::new(); + let mut found = false; + + match expr { + // This parses `sharding_key = 5`. But it's technically + // legal to write `5 = sharding_key`. I don't judge the people + // who do that, but I think ORMs will still use the first variant, + // so we can leave the second as a TODO. + Expr::BinaryOp { left, op, right } => { + match &**left { + Expr::BinaryOp { .. } => result.extend(self.selection_parser(&left)), + Expr::Identifier(ident) => { + found = ident.value + == *self.pool_settings.automatic_sharding_key.as_ref().unwrap(); + } + _ => (), + }; + + match op { + BinaryOperator::Eq => (), + BinaryOperator::Or => (), + BinaryOperator::And => (), + _ => { + // TODO: support other operators than equality. + debug!("Unsupported operation: {:?}", op); + return Vec::new(); + } + }; + + match &**right { + Expr::BinaryOp { .. } => result.extend(self.selection_parser(&right)), + Expr::Value(Value::Number(value, ..)) => { + if found { + match value.parse::() { + Ok(value) => result.push(value), + Err(_) => { + debug!("Sharding key was not an integer: {}", value); + } + }; + } + } + _ => (), + }; + } + + _ => (), + }; + + debug!("Sharding keys found: {:?}", result); + + result + } + + /// Try to figure out which shard the query should go to. + fn infer_shard(&self, query: &sqlparser::ast::Query) -> Option { + let mut shards = BTreeSet::new(); + + match &*query.body { + SetExpr::Query(query) => { + match self.infer_shard(&*query) { + Some(shard) => { + shards.insert(shard); + } + None => (), + }; + } + + SetExpr::Select(select) => { + match &select.selection { + Some(selection) => { + let sharding_keys = self.selection_parser(&selection); + + // TODO: Add support for prepared statements here. + // This should just give us the position of the value in the `B` message. + + let sharder = Sharder::new( + self.pool_settings.shards, + self.pool_settings.sharding_function, + ); + + for value in sharding_keys { + let shard = sharder.shard(value); + shards.insert(shard); + } + } + + None => (), + }; + } + _ => (), + }; + + match shards.len() { + // Didn't find a sharding key, you're on your own. + 0 => { + debug!("No sharding keys found"); + None + } + + 1 => Some(shards.into_iter().last().unwrap()), + + // TODO: support querying multiple shards (some day...) + _ => { + debug!("More than one sharding key found"); + None + } + } + } + /// Get the current desired server role we should be talking to. pub fn role(&self) -> Option { self.active_role @@ -392,7 +521,7 @@ mod test { } #[test] - fn test_infer_role_replica() { + fn test_infer_replica() { QueryRouter::setup(); let mut qr = QueryRouter::new(); assert!(qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")) != None); @@ -410,13 +539,13 @@ mod test { for query in queries { // It's a recognized query - assert!(qr.infer_role(query)); + assert!(qr.infer(query)); assert_eq!(qr.role(), Some(Role::Replica)); } } #[test] - fn test_infer_role_primary() { + fn test_infer_primary() { QueryRouter::setup(); let mut qr = QueryRouter::new(); @@ -429,24 +558,24 @@ mod test { for query in queries { // It's a recognized query - assert!(qr.infer_role(query)); + assert!(qr.infer(query)); assert_eq!(qr.role(), Some(Role::Primary)); } } #[test] - fn test_infer_role_primary_reads_enabled() { + fn test_infer_primary_reads_enabled() { QueryRouter::setup(); let mut qr = QueryRouter::new(); let query = simple_query("SELECT * FROM items WHERE id = 5"); assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO on")) != None); - assert!(qr.infer_role(query)); + assert!(qr.infer(query)); assert_eq!(qr.role(), None); } #[test] - fn test_infer_role_parse_prepared() { + fn test_infer_parse_prepared() { QueryRouter::setup(); let mut qr = QueryRouter::new(); qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")); @@ -461,7 +590,7 @@ mod test { res.put(prepared_stmt); res.put_i16(0); - assert!(qr.infer_role(res)); + assert!(qr.infer(res)); assert_eq!(qr.role(), Some(Role::Replica)); } @@ -625,11 +754,11 @@ mod test { assert_eq!(qr.role(), None); let query = simple_query("INSERT INTO test_table VALUES (1)"); - assert_eq!(qr.infer_role(query), true); + assert_eq!(qr.infer(query), true); assert_eq!(qr.role(), Some(Role::Primary)); let query = simple_query("SELECT * FROM test_table"); - assert_eq!(qr.infer_role(query), true); + assert_eq!(qr.infer(query), true); assert_eq!(qr.role(), Some(Role::Replica)); assert!(qr.query_parser_enabled()); @@ -644,12 +773,13 @@ mod test { let pool_settings = PoolSettings { pool_mode: PoolMode::Transaction, - shards: 0, + shards: 2, user: crate::config::User::default(), default_role: Some(Role::Replica), query_parser_enabled: true, primary_reads_enabled: false, sharding_function: ShardingFunction::PgBigintHash, + automatic_sharding_key: Some(String::from("id")), }; let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); @@ -672,6 +802,11 @@ mod test { let q2 = simple_query("SET SERVER ROLE TO 'default'"); assert!(qr.try_execute_command(q2) != None); assert_eq!(qr.active_role.unwrap(), pool_settings.clone().default_role); + + // Here we go :) + let q3 = simple_query("SELECT * FROM test WHERE id = 5 AND values IN (1, 2, 3)"); + assert!(qr.infer(q3)); + assert_eq!(qr.shard(), 1); } #[test] @@ -679,13 +814,13 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); - assert!(qr.infer_role(simple_query("BEGIN; SELECT 1; COMMIT;"))); + assert!(qr.infer(simple_query("BEGIN; SELECT 1; COMMIT;"))); assert_eq!(qr.role(), Role::Primary); - assert!(qr.infer_role(simple_query("SELECT 1; SELECT 2;"))); + assert!(qr.infer(simple_query("SELECT 1; SELECT 2;"))); assert_eq!(qr.role(), Role::Replica); - assert!(qr.infer_role(simple_query( + assert!(qr.infer(simple_query( "SELECT 123; INSERT INTO t VALUES (5); SELECT 1;" ))); assert_eq!(qr.role(), Role::Primary); From 5b6e641e0de30f725dbac80d6a31a647ca13699c Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 26 Oct 2022 00:14:08 -0700 Subject: [PATCH 228/551] Dont change shard unless you know (#195) --- src/query_router.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/query_router.rs b/src/query_router.rs index f65a77342..3f8e10619 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -335,8 +335,14 @@ impl QueryRouter { // or discard shard selection. If they point to the same shard though, // we can let them through as-is. // This is basically building a database now :) - self.active_shard = self.infer_shard(query); - debug!("Automatically using shard: {:?}", self.active_shard); + match self.infer_shard(query) { + Some(shard) => { + self.active_shard = Some(shard); + debug!("Automatically using shard: {:?}", self.active_shard); + } + + None => (), + }; } None => (), From 1e36aa64a4bdfe5a3f391136a775afd7bc94503b Mon Sep 17 00:00:00 2001 From: Pradeep Chhetri <30620077+chhetripradeep@users.noreply.github.com> Date: Tue, 1 Nov 2022 00:47:33 +0800 Subject: [PATCH 229/551] Add dependabot for keeping dependencies up-to-date (#196) It will help us to keep the dependencies up-to-date Signed-off-by: Pradeep Chhetri Signed-off-by: Pradeep Chhetri --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..f3202281f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "daily" + time: "04:00" # UTC + labels: + - "domain: deps" + commit-message: + prefix: "chore(deps)" + open-pull-requests-limit: 10 From a95867d2d14f4d176b120a362d37c14dfa5b4934 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:50:35 -0700 Subject: [PATCH 230/551] chore(deps): bump regex from 1.5.5 to 1.6.0 (#197) Bumps [regex](https://github.com/rust-lang/regex) from 1.5.5 to 1.6.0. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.5.5...1.6.0) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b2548325..7aaa845d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -724,9 +724,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.5" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" dependencies = [ "aho-corasick", "memchr", @@ -735,9 +735,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" [[package]] name = "ring" From 2309552a49e442c398ba28582356fea195358a55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:52:46 -0700 Subject: [PATCH 231/551] chore(deps): bump serde_derive from 1.0.136 to 1.0.147 (#198) Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.136 to 1.0.147. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.136...v1.0.147) --- updated-dependencies: - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7aaa845d1..c83250c48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -657,11 +657,11 @@ checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] @@ -799,9 +799,9 @@ checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" dependencies = [ "proc-macro2", "quote", @@ -900,13 +900,13 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.86" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -1064,6 +1064,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + [[package]] name = "unicode-normalization" version = "0.1.19" @@ -1073,12 +1079,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-xid" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" - [[package]] name = "untrusted" version = "0.7.1" From c3b89b93c748a328c1967465dd5d134ddff26a6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:53:04 -0700 Subject: [PATCH 232/551] chore(deps): bump sha2 from 0.10.2 to 0.10.5 (#199) Bumps [sha2](https://github.com/RustCrypto/hashes) from 0.10.2 to 0.10.5. - [Release notes](https://github.com/RustCrypto/hashes/releases) - [Commits](https://github.com/RustCrypto/hashes/compare/sha2-v0.10.2...sha2-v0.10.5) --- updated-dependencies: - dependency-name: sha2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c83250c48..2bb97a715 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -821,9 +821,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.2" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5" dependencies = [ "cfg-if", "cpufeatures", From e32001f4d4484b19986a577b4318f52f2053d497 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:53:19 -0700 Subject: [PATCH 233/551] chore(deps): bump async-trait from 0.1.52 to 0.1.58 (#200) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.52 to 0.1.58. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/compare/0.1.52...0.1.58) --- updated-dependencies: - dependency-name: async-trait dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2bb97a715..9273906ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" [[package]] name = "async-trait" -version = "0.1.52" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" dependencies = [ "proc-macro2", "quote", From 76c5da20b7ca1b223f650fd1d9e1b4f44bfeac71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:19:47 -0700 Subject: [PATCH 234/551] chore(deps): bump bytes from 1.1.0 to 1.2.1 (#206) Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.1.0 to 1.2.1. - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/compare/v1.1.0...v1.2.1) --- updated-dependencies: - dependency-name: bytes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9273906ad..57d216442 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,9 +87,9 @@ checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" [[package]] name = "bytes" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" [[package]] name = "cc" From 7f668aa806117dc6299f0c624ac5ea1f5bf21537 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:20:00 -0700 Subject: [PATCH 235/551] chore(deps): bump rustls-pemfile from 1.0.0 to 1.0.1 (#205) Bumps [rustls-pemfile](https://github.com/rustls/pemfile) from 1.0.0 to 1.0.1. - [Release notes](https://github.com/rustls/pemfile/releases) - [Commits](https://github.com/rustls/pemfile/commits) --- updated-dependencies: - dependency-name: rustls-pemfile dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57d216442..5744703b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -768,9 +768,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" dependencies = [ "base64", ] From 9c7abbcb071a5da31516bff1e84e571c8ff82ba0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:20:10 -0700 Subject: [PATCH 236/551] chore(deps): bump base64 from 0.13.0 to 0.13.1 (#203) Bumps [base64](https://github.com/marshallpierce/rust-base64) from 0.13.0 to 0.13.1. - [Release notes](https://github.com/marshallpierce/rust-base64/releases) - [Changelog](https://github.com/marshallpierce/rust-base64/blob/master/RELEASE-NOTES.md) - [Commits](https://github.com/marshallpierce/rust-base64/compare/v0.13.0...v0.13.1) --- updated-dependencies: - dependency-name: base64 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5744703b7..17d5ba684 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,9 +47,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "bb8" From 05c721f7b7562c14e2fa666ed7c3df0099873ef1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:20:20 -0700 Subject: [PATCH 237/551] chore(deps): bump arc-swap from 1.5.0 to 1.5.1 (#202) Bumps [arc-swap](https://github.com/vorner/arc-swap) from 1.5.0 to 1.5.1. - [Release notes](https://github.com/vorner/arc-swap/releases) - [Changelog](https://github.com/vorner/arc-swap/blob/master/CHANGELOG.md) - [Commits](https://github.com/vorner/arc-swap/compare/v1.5.0...v1.5.1) --- updated-dependencies: - dependency-name: arc-swap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17d5ba684..d236bed20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" +checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" [[package]] name = "async-trait" From f1251cde857c9214107bcdad24e39c4fa51ec805 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:20:30 -0700 Subject: [PATCH 238/551] chore(deps): bump rand from 0.8.4 to 0.8.5 (#201) Bumps [rand](https://github.com/rust-random/rand) from 0.8.4 to 0.8.5. - [Release notes](https://github.com/rust-random/rand/releases) - [Changelog](https://github.com/rust-random/rand/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/rand/compare/0.8.4...0.8.5) --- updated-dependencies: - dependency-name: rand dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d236bed20..3569d28bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -675,14 +675,13 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", "rand_core", - "rand_hc", ] [[package]] @@ -704,15 +703,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_hc" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" -dependencies = [ - "rand_core", -] - [[package]] name = "redox_syscall" version = "0.2.10" From c2b3702e9c4ff4072258935c4cfcdad40153c04f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 22:17:03 -0700 Subject: [PATCH 239/551] chore(deps): bump tokio from 1.16.1 to 1.19.2 (#210) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.16.1 to 1.19.2. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.16.1...tokio-1.19.2) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 153 ++++++++++++++++++++++++++++------------------------- 1 file changed, 82 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3569d28bf..8db0fa5d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,7 +60,7 @@ dependencies = [ "async-trait", "futures-channel", "futures-util", - "parking_lot 0.12.1", + "parking_lot", "tokio", ] @@ -236,7 +236,7 @@ checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.10.0+wasi-snapshot-preview1", ] [[package]] @@ -356,15 +356,6 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - [[package]] name = "itoa" version = "1.0.3" @@ -448,33 +439,14 @@ checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "mio" -version = "0.7.14" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", - "miow", - "ntapi", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", -] - -[[package]] -name = "ntapi" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" -dependencies = [ - "winapi", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.42.0", ] [[package]] @@ -512,17 +484,6 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.5", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -530,21 +491,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.3", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -557,7 +504,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.36.1", ] [[package]] @@ -579,7 +526,7 @@ dependencies = [ "md-5", "num_cpus", "once_cell", - "parking_lot 0.12.1", + "parking_lot", "phf", "rand", "regex", @@ -915,7 +862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", - "wasi", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] @@ -936,9 +883,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.16.1" +version = "1.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c27a64b625de6d309e8c57716ba93021dccf1b3b5c97edd6d3dd2d2135afc0a" +checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" dependencies = [ "bytes", "libc", @@ -946,9 +893,10 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot 0.11.2", + "parking_lot", "pin-project-lite", "signal-hook-registry", + "socket2", "tokio-macros", "winapi", ] @@ -1097,6 +1045,12 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.81" @@ -1208,39 +1162,96 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + [[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + [[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" From 87b43db40d798500142d615601e16526bdcdb22c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 22:17:29 -0700 Subject: [PATCH 240/551] chore(deps): bump once_cell from 1.9.0 to 1.16.0 (#209) Bumps [once_cell](https://github.com/matklad/once_cell) from 1.9.0 to 1.16.0. - [Release notes](https://github.com/matklad/once_cell/releases) - [Changelog](https://github.com/matklad/once_cell/blob/master/CHANGELOG.md) - [Commits](https://github.com/matklad/once_cell/compare/v1.9.0...v1.16.0) --- updated-dependencies: - dependency-name: once_cell dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8db0fa5d5..0c83140f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -480,9 +480,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.9.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" [[package]] name = "parking_lot" From f0180a78ad95d65dbcd8a4bd3eb4b075257516d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 23:35:13 -0700 Subject: [PATCH 241/551] chore(deps): bump log from 0.4.14 to 0.4.17 (#216) Bumps [log](https://github.com/rust-lang/log) from 0.4.14 to 0.4.17. - [Release notes](https://github.com/rust-lang/log/releases) - [Changelog](https://github.com/rust-lang/log/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/log/compare/0.4.14...0.4.17) --- updated-dependencies: - dependency-name: log dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c83140f3..02f99a903 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] From 46ba5f9b8fdabfe3555fb6b716221c35a477829f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 23:35:23 -0700 Subject: [PATCH 242/551] chore(deps): bump serde from 1.0.136 to 1.0.147 (#215) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.136 to 1.0.147. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.136...v1.0.147) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 02f99a903..8a16af596 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -730,9 +730,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.136" +version = "1.0.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" [[package]] name = "serde_derive" From c9d62ce51812e12b0da22b0c0faec1a8a9ed5023 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 23:35:30 -0700 Subject: [PATCH 243/551] chore(deps): bump md-5 from 0.10.0 to 0.10.4 (#214) Bumps [md-5](https://github.com/RustCrypto/hashes) from 0.10.0 to 0.10.4. - [Release notes](https://github.com/RustCrypto/hashes/releases) - [Commits](https://github.com/RustCrypto/hashes/compare/md2-v0.10.0...md-5-v0.10.4) --- updated-dependencies: - dependency-name: md-5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a16af596..b023f593f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -424,9 +424,9 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.0" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6a38fc55c8bbc10058782919516f88826e70320db6d206aebc49611d24216ae" +checksum = "66b48670c893079d3c2ed79114e3644b7004df1c361a4e0ad52e2e6940d07c3d" dependencies = [ "digest", ] From 059e14453475f4a33ff3651f37239c58db4c02a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 23:35:37 -0700 Subject: [PATCH 244/551] chore(deps): bump sha2 from 0.10.5 to 0.10.6 (#213) Bumps [sha2](https://github.com/RustCrypto/hashes) from 0.10.5 to 0.10.6. - [Release notes](https://github.com/RustCrypto/hashes/releases) - [Commits](https://github.com/RustCrypto/hashes/compare/sha2-v0.10.5...sha2-v0.10.6) --- updated-dependencies: - dependency-name: sha2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b023f593f..b9bd3925f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,9 +137,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ "block-buffer", "crypto-common", @@ -758,9 +758,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", From 58a965fbe3bea1cb8d5a5e34e26adcd92372d6a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 23:35:45 -0700 Subject: [PATCH 245/551] chore(deps): bump env_logger from 0.9.0 to 0.9.1 (#212) Bumps [env_logger](https://github.com/env-logger-rs/env_logger) from 0.9.0 to 0.9.1. - [Release notes](https://github.com/env-logger-rs/env_logger/releases) - [Changelog](https://github.com/env-logger-rs/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/env-logger-rs/env_logger/compare/v0.9.0...v0.9.1) --- updated-dependencies: - dependency-name: env_logger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9bd3925f..71ff90ca1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" dependencies = [ "atty", "humantime", From 72954a54c06769398dba248ef6e7278b791331a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 23:35:50 -0700 Subject: [PATCH 246/551] chore(deps): bump chrono from 0.4.19 to 0.4.22 (#211) Bumps [chrono](https://github.com/chronotope/chrono) from 0.4.19 to 0.4.22. - [Release notes](https://github.com/chronotope/chrono/releases) - [Changelog](https://github.com/chronotope/chrono/blob/v0.4.22/CHANGELOG.md) - [Commits](https://github.com/chronotope/chrono/compare/v0.4.19...v0.4.22) --- updated-dependencies: - dependency-name: chrono dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 119 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71ff90ca1..01ef126d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "arc-swap" version = "1.5.1" @@ -105,17 +114,35 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" dependencies = [ - "libc", + "iana-time-zone", + "js-sys", "num-integer", "num-traits", "time", + "wasm-bindgen", "winapi", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cpufeatures" version = "0.2.1" @@ -135,6 +162,50 @@ dependencies = [ "typenum", ] +[[package]] +name = "cxx" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.10.5" @@ -346,6 +417,30 @@ dependencies = [ "want", ] +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "indexmap" version = "1.9.1" @@ -404,6 +499,15 @@ version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" +[[package]] +name = "link-cplusplus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + [[package]] name = "lock_api" version = "0.4.6" @@ -718,6 +822,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + [[package]] name = "sct" version = "0.7.0" @@ -1017,6 +1127,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "untrusted" version = "0.7.1" From 8573661560a85498a60ab78e8b4c44b21ee64b3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 23:35:54 -0700 Subject: [PATCH 247/551] chore(deps): bump toml from 0.5.8 to 0.5.9 (#207) Bumps [toml](https://github.com/alexcrichton/toml-rs) from 0.5.8 to 0.5.9. - [Release notes](https://github.com/alexcrichton/toml-rs/releases) - [Commits](https://github.com/alexcrichton/toml-rs/compare/0.5.8...0.5.9) --- updated-dependencies: - dependency-name: toml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 01ef126d7..d58acb704 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1049,9 +1049,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] From 8bb7b4305af6b937205f8461d42ae3a18ef0f7c0 Mon Sep 17 00:00:00 2001 From: Pradeep Chhetri <30620077+chhetripradeep@users.noreply.github.com> Date: Wed, 2 Nov 2022 23:11:47 +0800 Subject: [PATCH 248/551] Fix for warnings about avg_errors not implemented (#220) --- src/prometheus.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/prometheus.rs b/src/prometheus.rs index d54ba3291..2d84cb54c 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -63,6 +63,10 @@ static METRIC_HELP_AND_TYPES_LOOKUP: phf::Map<&'static str, MetricHelpType> = ph help: "Average of total_sent bytes every 15 seconds", ty: "gauge", }, + "avg_errors" => MetricHelpType { + help: "Average number of errors every 15 seconds", + ty: "gauge", + }, "avg_xact_count" => MetricHelpType { help: "Average of total_xact_count every 15 seconds", ty: "gauge", From 33deb2ba58f419579b022e39be8ea6ef12ee5f8c Mon Sep 17 00:00:00 2001 From: Pradeep Chhetri <30620077+chhetripradeep@users.noreply.github.com> Date: Thu, 3 Nov 2022 03:10:12 +0800 Subject: [PATCH 249/551] Fix dependabot labels for pull-requests (#219) --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f3202281f..2dd575258 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,7 @@ updates: interval: "daily" time: "04:00" # UTC labels: - - "domain: deps" + - "dependencies" commit-message: prefix: "chore(deps)" open-pull-requests-limit: 10 From 688418e710264970cd79dd556ddd351793cafb98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 12:10:41 -0700 Subject: [PATCH 250/551] chore(deps): bump jemallocator from 0.3.2 to 0.5.0 (#218) Bumps [jemallocator](https://github.com/tikv/jemallocator) from 0.3.2 to 0.5.0. - [Release notes](https://github.com/tikv/jemallocator/releases) - [Changelog](https://github.com/tikv/jemallocator/blob/main/CHANGELOG.md) - [Commits](https://github.com/tikv/jemallocator/commits/0.5.0) --- updated-dependencies: - dependency-name: jemallocator dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d58acb704..890360a23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -459,9 +459,9 @@ checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" [[package]] name = "jemalloc-sys" -version = "0.3.2" +version = "0.5.2+5.3.0-patched" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" +checksum = "134163979b6eed9564c98637b710b40979939ba351f59952708234ea11b5f3f8" dependencies = [ "cc", "fs_extra", @@ -470,9 +470,9 @@ dependencies = [ [[package]] name = "jemallocator" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +checksum = "16c2514137880c52b0b4822b563fadd38257c1f380858addb74a400889696ea6" dependencies = [ "jemalloc-sys", "libc", diff --git a/Cargo.toml b/Cargo.toml index e06b6930a..05a8bb28b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,4 +36,4 @@ phf = { version = "0.11.1", features = ["macros"] } exitcode = "1.1.2" [target.'cfg(not(target_env = "msvc"))'.dependencies] -jemallocator = "0.3.2" +jemallocator = "0.5.0" From cd3ba46fd0f4ec260640a90b0f6ad93ffc2ca041 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Nov 2022 12:10:55 -0700 Subject: [PATCH 251/551] chore(deps): bump md-5 from 0.10.4 to 0.10.5 (#217) Bumps [md-5](https://github.com/RustCrypto/hashes) from 0.10.4 to 0.10.5. - [Release notes](https://github.com/RustCrypto/hashes/releases) - [Commits](https://github.com/RustCrypto/hashes/compare/md-5-v0.10.4...md-5-v0.10.5) --- updated-dependencies: - dependency-name: md-5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 890360a23..4f9466b4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -528,9 +528,9 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b48670c893079d3c2ed79114e3644b7004df1c361a4e0ad52e2e6940d07c3d" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ "digest", ] From ba596acfdbecf4567c27d131f8180cf8a1b773e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 5 Nov 2022 10:20:55 -0700 Subject: [PATCH 252/551] chore(deps): bump num_cpus from 1.13.1 to 1.14.0 (#221) Bumps [num_cpus](https://github.com/seanmonstar/num_cpus) from 1.13.1 to 1.14.0. - [Release notes](https://github.com/seanmonstar/num_cpus/releases) - [Changelog](https://github.com/seanmonstar/num_cpus/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/num_cpus/compare/v1.13.1...v1.14.0) --- updated-dependencies: - dependency-name: num_cpus dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f9466b4d..dbcbb6a7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,9 +574,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ "hermit-abi", "libc", From 27417e4573182a2acac99bd6f09128f4e2743b2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 08:07:12 -0800 Subject: [PATCH 253/551] chore(deps): bump regex from 1.6.0 to 1.7.0 (#224) Bumps [regex](https://github.com/rust-lang/regex) from 1.6.0 to 1.7.0. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.6.0...1.7.0) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dbcbb6a7a..e33993c8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,9 +765,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "aho-corasick", "memchr", From 037eb0cf5d8a9fe85e61fe692a38f8d9335e1ccd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 08:07:53 -0800 Subject: [PATCH 254/551] chore(deps): bump env_logger from 0.9.1 to 0.9.3 (#223) Bumps [env_logger](https://github.com/env-logger-rs/env_logger) from 0.9.1 to 0.9.3. - [Release notes](https://github.com/env-logger-rs/env_logger/releases) - [Changelog](https://github.com/env-logger-rs/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/env-logger-rs/env_logger/compare/v0.9.1...v0.9.3) --- updated-dependencies: - dependency-name: env_logger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e33993c8a..2c0819d5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -219,9 +219,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" dependencies = [ "atty", "humantime", From 4ba66c9302085a9a50f729021e5a4df2fea5f29d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 08:08:46 -0800 Subject: [PATCH 255/551] chore(deps): bump hyper from 0.14.20 to 0.14.23 (#222) Bumps [hyper](https://github.com/hyperium/hyper) from 0.14.20 to 0.14.23. - [Release notes](https://github.com/hyperium/hyper/releases) - [Changelog](https://github.com/hyperium/hyper/blob/v0.14.23/CHANGELOG.md) - [Commits](https://github.com/hyperium/hyper/compare/v0.14.20...v0.14.23) --- updated-dependencies: - dependency-name: hyper dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c0819d5e..9ccf29442 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -377,9 +377,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" @@ -395,9 +395,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.20" +version = "0.14.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" dependencies = [ "bytes", "futures-channel", @@ -906,9 +906,9 @@ checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "socket2" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" dependencies = [ "libc", "winapi", From 66d06fa2a1b29ce2de9289789bfa8c31cb21c285 Mon Sep 17 00:00:00 2001 From: Cluas Date: Thu, 10 Nov 2022 02:04:31 +0800 Subject: [PATCH 256/551] chore: make clippy lint happy (#225) * chore: make clippy happy * chore: cargo fmt * chore: cargo fmt --- src/admin.rs | 4 +- src/client.rs | 41 ++++++++-------- src/config.rs | 31 +++++------- src/main.rs | 23 ++------- src/messages.rs | 34 ++++++------- src/pool.rs | 46 ++++++++---------- src/prometheus.rs | 2 +- src/query_router.rs | 115 ++++++++++++++++++++------------------------ src/scram.rs | 16 +++--- src/server.rs | 22 ++++----- src/sharding.rs | 4 +- src/stats.rs | 32 ++++++------ src/tls.rs | 4 +- 13 files changed, 167 insertions(+), 207 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 42af315ed..d4979fdb5 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -22,7 +22,7 @@ pub fn generate_server_info_for_admin() -> BytesMut { server_info.put(server_parameter_message("server_version", VERSION)); server_info.put(server_parameter_message("DateStyle", "ISO, MDY")); - return server_info; + server_info } /// Handle admin client. @@ -179,7 +179,7 @@ where let mut res = BytesMut::new(); res.put(row_description(&vec![("version", DataType::Text)])); - res.put(data_row(&vec![format!("PgCat {}", VERSION).to_string()])); + res.put(data_row(&vec![format!("PgCat {}", VERSION)])); res.put(command_complete("SHOW")); res.put_u8(b'Z'); diff --git a/src/client.rs b/src/client.rs index 7db920525..92e60150f 100644 --- a/src/client.rs +++ b/src/client.rs @@ -377,7 +377,7 @@ where let admin = ["pgcat", "pgbouncer"] .iter() - .filter(|db| *db == &pool_name) + .filter(|db| *db == pool_name) .count() == 1; @@ -389,7 +389,7 @@ where ); error_response_terminal( &mut write, - &format!("terminating connection due to administrator command"), + "terminating connection due to administrator command", ) .await?; return Err(Error::ShuttingDown); @@ -446,7 +446,7 @@ where } // Authenticate normal user. else { - let pool = match get_pool(&pool_name, &username) { + let pool = match get_pool(pool_name, username) { Some(pool) => pool, None => { error_response( @@ -464,7 +464,7 @@ where }; // Compare server and client hashes. - let password_hash = md5_hash_password(&username, &pool.settings.user.password, &salt); + let password_hash = md5_hash_password(username, &pool.settings.user.password, &salt); if password_hash != password_response { warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name); @@ -487,9 +487,9 @@ where trace!("Startup OK"); - return Ok(Client { + Ok(Client { read: BufReader::new(read), - write: write, + write, addr, buffer: BytesMut::with_capacity(8196), cancel_mode: false, @@ -498,8 +498,8 @@ where secret_key, client_server_map, parameters: parameters.clone(), - stats: stats, - admin: admin, + stats, + admin, last_address_id: None, last_server_id: None, pool_name: pool_name.clone(), @@ -507,7 +507,7 @@ where application_name: application_name.to_string(), shutdown, connected_to_server: false, - }); + }) } /// Handle cancel request. @@ -521,9 +521,9 @@ where ) -> Result, Error> { let process_id = bytes.get_i32(); let secret_key = bytes.get_i32(); - return Ok(Client { + Ok(Client { read: BufReader::new(read), - write: write, + write, addr, buffer: BytesMut::with_capacity(8196), cancel_mode: true, @@ -541,7 +541,7 @@ where application_name: String::from("undefined"), shutdown, connected_to_server: false, - }); + }) } /// Handle a connected and authenticated client. @@ -557,12 +557,9 @@ where // Drop the mutex as soon as possible. // We found the server the client is using for its query // that it wants to cancel. - Some((process_id, secret_key, address, port)) => ( - process_id.clone(), - secret_key.clone(), - address.clone(), - *port, - ), + Some((process_id, secret_key, address, port)) => { + (*process_id, *secret_key, address.clone(), *port) + } // The client doesn't know / got the wrong server, // we're closing the connection for security reasons. @@ -573,7 +570,7 @@ where // Opens a new separate connection to the server, sends the backend_id // and secret_key and then closes it for security reasons. No other interactions // take place. - return Ok(Server::cancel(&address, port, process_id, secret_key).await?); + return Server::cancel(&address, port, process_id, secret_key).await; } // The query router determines where the query is going to go, @@ -606,7 +603,7 @@ where if !self.admin { error_response_terminal( &mut self.write, - &format!("terminating connection due to administrator command") + "terminating connection due to administrator command" ).await?; return Ok(()) } @@ -998,14 +995,14 @@ where ) -> Result<(), Error> { debug!("Sending {} to server", code); - self.send_server_message(server, message, &address, &pool) + self.send_server_message(server, message, address, pool) .await?; let query_start = Instant::now(); // Read all data the server has to offer, which can be multiple messages // buffered in 8196 bytes chunks. loop { - let response = self.receive_server_message(server, &address, &pool).await?; + let response = self.receive_server_message(server, address, pool).await?; match write_all_half(&mut self.write, response).await { Ok(_) => (), diff --git a/src/config.rs b/src/config.rs index 647b0166c..8e1e14eeb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -9,7 +9,6 @@ use std::path::Path; use std::sync::Arc; use tokio::fs::File; use tokio::io::AsyncReadExt; -use toml; use crate::errors::Error; use crate::pool::{ClientServerMap, ConnectionPool}; @@ -353,7 +352,7 @@ impl Shard { let mut dup_check = HashSet::new(); let mut primary_count = 0; - if self.servers.len() == 0 { + if self.servers.is_empty() { error!("Shard {} has no servers configured", self.database); return Err(Error::BadConfig); } @@ -362,10 +361,9 @@ impl Shard { dup_check.insert(server); // Check that we define only zero or one primary. - match server.role { - Role::Primary => primary_count += 1, - _ => (), - }; + if server.role == Role::Primary { + primary_count += 1 + } } if primary_count > 1 { @@ -605,22 +603,17 @@ impl Config { // Validate TLS! match self.general.tls_certificate.clone() { Some(tls_certificate) => { - match load_certs(&Path::new(&tls_certificate)) { + match load_certs(Path::new(&tls_certificate)) { Ok(_) => { // Cert is okay, but what about the private key? match self.general.tls_private_key.clone() { - Some(tls_private_key) => { - match load_keys(&Path::new(&tls_private_key)) { - Ok(_) => (), - Err(err) => { - error!( - "tls_private_key is incorrectly configured: {:?}", - err - ); - return Err(Error::BadConfig); - } + Some(tls_private_key) => match load_keys(Path::new(&tls_private_key)) { + Ok(_) => (), + Err(err) => { + error!("tls_private_key is incorrectly configured: {:?}", err); + return Err(Error::BadConfig); } - } + }, None => { error!("tls_certificate is set, but the tls_private_key is not"); @@ -638,7 +631,7 @@ impl Config { None => (), }; - for (_, pool) in &mut self.pools { + for pool in self.pools.values_mut() { pool.validate()?; } diff --git a/src/main.rs b/src/main.rs index 75aab69b9..2572d9f3f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -75,7 +75,6 @@ mod stats; mod tls; use crate::config::{get_config, reload_config, VERSION}; -use crate::errors::Error; use crate::pool::{ClientServerMap, ConnectionPool}; use crate::prometheus::start_metric_server; use crate::stats::{Collector, Reporter, REPORTER}; @@ -171,13 +170,10 @@ async fn main() { if config.general.autoreload { info!("Automatically reloading config"); - match reload_config(autoreload_client_server_map.clone()).await { - Ok(changed) => { - if changed { - get_config().show() - } + if let Ok(changed) = reload_config(autoreload_client_server_map.clone()).await { + if changed { + get_config().show() } - Err(_) => (), }; } } @@ -202,10 +198,7 @@ async fn main() { _ = sighup_signal.recv() => { info!("Reloading config"); - match reload_config(client_server_map.clone()).await { - Ok(_) => (), - Err(_) => (), - }; + _ = reload_config(client_server_map.clone()).await; get_config().show(); }, @@ -278,14 +271,6 @@ async fn main() { } Err(err) => { - match err { - // Don't count the clients we rejected. - Error::ShuttingDown => (), - _ => { - // drain_tx.send(-1).await.unwrap(); - } - } - warn!("Client disconnected with error {:?}", err); } }; diff --git a/src/messages.rs b/src/messages.rs index 78cb9dbf7..0d7bc5759 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -38,7 +38,7 @@ where auth_ok.put_i32(8); auth_ok.put_i32(0); - Ok(write_all(stream, auth_ok).await?) + write_all(stream, auth_ok).await } /// Generate md5 password challenge. @@ -79,7 +79,7 @@ where key_data.put_i32(backend_id); key_data.put_i32(secret_key); - Ok(write_all(stream, key_data).await?) + write_all(stream, key_data).await } /// Construct a `Q`: Query message. @@ -88,7 +88,7 @@ pub fn simple_query(query: &str) -> BytesMut { let query = format!("{}\0", query); res.put_i32(query.len() as i32 + 4); - res.put_slice(&query.as_bytes()); + res.put_slice(query.as_bytes()); res } @@ -106,7 +106,7 @@ where bytes.put_i32(5); bytes.put_u8(b'I'); // Idle - Ok(write_all(stream, bytes).await?) + write_all(stream, bytes).await } /// Send the startup packet the server. We're pretending we're a Pg client. @@ -118,12 +118,12 @@ pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Resu // User bytes.put(&b"user\0"[..]); - bytes.put_slice(&user.as_bytes()); + bytes.put_slice(user.as_bytes()); bytes.put_u8(0); // Database bytes.put(&b"database\0"[..]); - bytes.put_slice(&database.as_bytes()); + bytes.put_slice(database.as_bytes()); bytes.put_u8(0); bytes.put_u8(0); // Null terminator @@ -136,7 +136,7 @@ pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Resu match stream.write_all(&startup).await { Ok(_) => Ok(()), - Err(_) => return Err(Error::SocketError), + Err(_) => Err(Error::SocketError), } } @@ -155,7 +155,7 @@ pub fn parse_params(mut bytes: BytesMut) -> Result, Erro c = bytes.get_u8(); } - if tmp.len() > 0 { + if !tmp.is_empty() { buf.push(tmp.clone()); tmp.clear(); } @@ -234,7 +234,7 @@ where message.put_i32(password.len() as i32 + 4); message.put_slice(&password[..]); - Ok(write_all(stream, message).await?) + write_all(stream, message).await } /// Implements a response to our custom `SET SHARDING KEY` @@ -292,7 +292,7 @@ where // The short error message. error.put_u8(b'M'); - error.put_slice(&format!("{}\0", message).as_bytes()); + error.put_slice(format!("{}\0", message).as_bytes()); // No more fields follow. error.put_u8(0); @@ -304,7 +304,7 @@ where res.put_i32(error.len() as i32 + 4); res.put(error); - Ok(write_all_half(stream, res).await?) + write_all_half(stream, res).await } pub async fn wrong_password(stream: &mut S, user: &str) -> Result<(), Error> @@ -327,7 +327,7 @@ where // The short error message. error.put_u8(b'M'); - error.put_slice(&format!("password authentication failed for user \"{}\"\0", user).as_bytes()); + error.put_slice(format!("password authentication failed for user \"{}\"\0", user).as_bytes()); // No more fields follow. error.put_u8(0); @@ -379,7 +379,7 @@ pub fn row_description(columns: &Vec<(&str, DataType)>) -> BytesMut { for (name, data_type) in columns { // Column name - row_desc.put_slice(&format!("{}\0", name).as_bytes()); + row_desc.put_slice(format!("{}\0", name).as_bytes()); // Doesn't belong to any table row_desc.put_i32(0); @@ -423,7 +423,7 @@ pub fn data_row(row: &Vec) -> BytesMut { for column in row { let column = column.as_bytes(); data_row.put_i32(column.len() as i32); - data_row.put_slice(&column); + data_row.put_slice(column); } res.put_u8(b'D'); @@ -450,7 +450,7 @@ where { match stream.write_all(&buf).await { Ok(_) => Ok(()), - Err(_) => return Err(Error::SocketError), + Err(_) => Err(Error::SocketError), } } @@ -461,7 +461,7 @@ where { match stream.write_all(&buf).await { Ok(_) => Ok(()), - Err(_) => return Err(Error::SocketError), + Err(_) => Err(Error::SocketError), } } @@ -510,5 +510,5 @@ pub fn server_parameter_message(key: &str, value: &str) -> BytesMut { server_info.put_slice(value.as_bytes()); server_info.put_bytes(0, 1); - return server_info; + server_info } diff --git a/src/pool.rs b/src/pool.rs index 2b80f7bab..07b7a40bd 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -140,18 +140,18 @@ impl ConnectionPool { let changed = pools_hash.insert(pool_config.clone()); // There is one pool per database/user pair. - for (_, user) in &pool_config.users { + for user in pool_config.users.values() { // If the pool hasn't changed, get existing reference and insert it into the new_pools. // We replace all pools at the end, but if the reference is kept, the pool won't get re-created (bb8). if !changed { - match get_pool(&pool_name, &user.username) { + match get_pool(pool_name, &user.username) { Some(pool) => { info!( "[pool: {}][user: {}] has not changed", pool_name, user.username ); new_pools.insert( - PoolIdentifier::new(&pool_name, &user.username), + PoolIdentifier::new(pool_name, &user.username), pool.clone(), ); continue; @@ -172,7 +172,6 @@ impl ConnectionPool { .shards .clone() .into_keys() - .map(|x| x.to_string()) .collect::>(); // Sort by shard number to ensure consistency. @@ -182,10 +181,9 @@ impl ConnectionPool { let shard = &pool_config.shards[shard_idx]; let mut pools = Vec::new(); let mut servers = Vec::new(); - let mut address_index = 0; let mut replica_number = 0; - for server in shard.servers.iter() { + for (address_index, server) in shard.servers.iter().enumerate() { let address = Address { id: address_id, database: shard.database.clone(), @@ -200,7 +198,6 @@ impl ConnectionPool { }; address_id += 1; - address_index += 1; if server.role == Role::Replica { replica_number += 1; @@ -240,7 +237,7 @@ impl ConnectionPool { let mut pool = ConnectionPool { databases: shards, - addresses: addresses, + addresses, banlist: Arc::new(RwLock::new(banlist)), stats: get_reporter(), server_info: BytesMut::new(), @@ -255,7 +252,7 @@ impl ConnectionPool { "primary" => Some(Role::Primary), _ => unreachable!(), }, - query_parser_enabled: pool_config.query_parser_enabled.clone(), + query_parser_enabled: pool_config.query_parser_enabled, primary_reads_enabled: pool_config.primary_reads_enabled, sharding_function: pool_config.sharding_function, automatic_sharding_key: pool_config.automatic_sharding_key.clone(), @@ -273,7 +270,7 @@ impl ConnectionPool { }; // There is one pool per database/user pair. - new_pools.insert(PoolIdentifier::new(&pool_name, &user.username), pool); + new_pools.insert(PoolIdentifier::new(pool_name, &user.username), pool); } } @@ -304,7 +301,7 @@ impl ConnectionPool { let server = &*proxy; let server_info = server.server_info(); - if server_infos.len() > 0 { + if !server_infos.is_empty() { // Compare against the last server checked. if server_info != server_infos[server_infos.len() - 1] { warn!( @@ -320,7 +317,7 @@ impl ConnectionPool { // TODO: compare server information to make sure // all shards are running identical configurations. - if server_infos.len() == 0 { + if server_infos.is_empty() { return Err(Error::AllServersDown); } @@ -356,7 +353,7 @@ impl ConnectionPool { None => break, }; - if self.is_banned(&address, role) { + if self.is_banned(address, role) { debug!("Address {:?} is banned", address); continue; } @@ -373,7 +370,7 @@ impl ConnectionPool { Ok(conn) => conn, Err(err) => { error!("Banning instance {:?}, error: {:?}", address, err); - self.ban(&address, process_id); + self.ban(address, process_id); self.stats.client_checkout_error(process_id, address.id); continue; } @@ -428,7 +425,7 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(&address, process_id); + self.ban(address, process_id); continue; } }, @@ -442,7 +439,7 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(&address, process_id); + self.ban(address, process_id); continue; } } @@ -575,11 +572,11 @@ impl ServerPool { stats: Reporter, ) -> ServerPool { ServerPool { - address: address, - user: user, + address, + user, database: database.to_string(), - client_server_map: client_server_map, - stats: stats, + client_server_map, + stats, } } } @@ -638,15 +635,14 @@ impl ManageConnection for ServerPool { /// Get the connection pool pub fn get_pool(db: &str, user: &str) -> Option { - match (*(*POOLS.load())).get(&PoolIdentifier::new(db, user)) { - Some(pool) => Some(pool.clone()), - None => None, - } + (*(*POOLS.load())) + .get(&PoolIdentifier::new(db, user)) + .cloned() } /// Get a pointer to all configured pools. pub fn get_all_pools() -> HashMap { - return (*(*POOLS.load())).clone(); + (*(*POOLS.load())).clone() } /// How many total servers we have in the config. diff --git a/src/prometheus.rs b/src/prometheus.rs index 2d84cb54c..ec011b396 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -203,7 +203,7 @@ async fn prometheus_stats(request: Request) -> Result, hype pub async fn start_metric_server(http_addr: SocketAddr) { let http_service_factory = make_service_fn(|_conn| async { Ok::<_, hyper::Error>(service_fn(prometheus_stats)) }); - let server = Server::bind(&http_addr.into()).serve(http_service_factory); + let server = Server::bind(&http_addr).serve(http_service_factory); info!( "Exposing prometheus metrics on http://{}/metrics.", http_addr diff --git a/src/query_router.rs b/src/query_router.rs index 3f8e10619..552c358c3 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -86,10 +86,7 @@ impl QueryRouter { Err(_) => return false, }; - match CUSTOM_SQL_REGEX_SET.set(set) { - Ok(_) => true, - Err(_) => false, - } + CUSTOM_SQL_REGEX_SET.set(set).is_ok() } /// Create a new instance of the query router. @@ -276,7 +273,6 @@ impl QueryRouter { // Parse (prepared statement) 'P' => { let mut start = 0; - let mut end; // Skip the name of the prepared statement. while buf[start] != 0 && start < buf.len() { @@ -285,7 +281,7 @@ impl QueryRouter { start += 1; // Skip terminating null // Find the end of the prepared stmt (\0) - end = start; + let mut end = start; while buf[end] != 0 && end < buf.len() { end += 1; } @@ -294,7 +290,7 @@ impl QueryRouter { debug!("Prepared statement: '{}'", query); - query.replace("$", "") // Remove placeholders turning them into "values" + query.replace('$', "") // Remove placeholders turning them into "values" } _ => return false, @@ -312,7 +308,7 @@ impl QueryRouter { debug!("AST: {:?}", ast); - if ast.len() == 0 { + if ast.is_empty() { // That's weird, no idea, let's go to primary self.active_role = Some(Role::Primary); return false; @@ -371,50 +367,46 @@ impl QueryRouter { let mut result = Vec::new(); let mut found = false; - match expr { - // This parses `sharding_key = 5`. But it's technically - // legal to write `5 = sharding_key`. I don't judge the people - // who do that, but I think ORMs will still use the first variant, - // so we can leave the second as a TODO. - Expr::BinaryOp { left, op, right } => { - match &**left { - Expr::BinaryOp { .. } => result.extend(self.selection_parser(&left)), - Expr::Identifier(ident) => { - found = ident.value - == *self.pool_settings.automatic_sharding_key.as_ref().unwrap(); - } - _ => (), - }; + // This parses `sharding_key = 5`. But it's technically + // legal to write `5 = sharding_key`. I don't judge the people + // who do that, but I think ORMs will still use the first variant, + // so we can leave the second as a TODO. + if let Expr::BinaryOp { left, op, right } = expr { + match &**left { + Expr::BinaryOp { .. } => result.extend(self.selection_parser(left)), + Expr::Identifier(ident) => { + found = + ident.value == *self.pool_settings.automatic_sharding_key.as_ref().unwrap(); + } + _ => (), + }; - match op { - BinaryOperator::Eq => (), - BinaryOperator::Or => (), - BinaryOperator::And => (), - _ => { - // TODO: support other operators than equality. - debug!("Unsupported operation: {:?}", op); - return Vec::new(); - } - }; + match op { + BinaryOperator::Eq => (), + BinaryOperator::Or => (), + BinaryOperator::And => (), + _ => { + // TODO: support other operators than equality. + debug!("Unsupported operation: {:?}", op); + return Vec::new(); + } + }; - match &**right { - Expr::BinaryOp { .. } => result.extend(self.selection_parser(&right)), - Expr::Value(Value::Number(value, ..)) => { - if found { - match value.parse::() { - Ok(value) => result.push(value), - Err(_) => { - debug!("Sharding key was not an integer: {}", value); - } - }; - } + match &**right { + Expr::BinaryOp { .. } => result.extend(self.selection_parser(right)), + Expr::Value(Value::Number(value, ..)) => { + if found { + match value.parse::() { + Ok(value) => result.push(value), + Err(_) => { + debug!("Sharding key was not an integer: {}", value); + } + }; } - _ => (), - }; - } - - _ => (), - }; + } + _ => (), + }; + } debug!("Sharding keys found: {:?}", result); @@ -438,7 +430,7 @@ impl QueryRouter { SetExpr::Select(select) => { match &select.selection { Some(selection) => { - let sharding_keys = self.selection_parser(&selection); + let sharding_keys = self.selection_parser(selection); // TODO: Add support for prepared statements here. // This should just give us the position of the value in the `B` message. @@ -484,10 +476,7 @@ impl QueryRouter { /// Get desired shard we should be talking to. pub fn shard(&self) -> usize { - match self.active_shard { - Some(shard) => shard, - None => 0, - } + self.active_shard.unwrap_or(0) } pub fn set_shard(&mut self, shard: usize) { @@ -531,7 +520,7 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); assert!(qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")) != None); - assert_eq!(qr.query_parser_enabled(), true); + assert!(qr.query_parser_enabled()); assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); @@ -656,9 +645,9 @@ mod test { for (i, test) in tests.iter().enumerate() { if !list[matches[i]].is_match(test) { println!("{} does not match {}", test, list[matches[i]]); - assert!(false); + panic!(); } - assert_eq!(set.matches(test).into_iter().collect::>().len(), 1); + assert_eq!(set.matches(test).into_iter().count(), 1); } let bad = [ @@ -667,7 +656,7 @@ mod test { ]; for query in &bad { - assert_eq!(set.matches(query).into_iter().collect::>().len(), 0); + assert_eq!(set.matches(query).into_iter().count(), 0); } } @@ -760,11 +749,11 @@ mod test { assert_eq!(qr.role(), None); let query = simple_query("INSERT INTO test_table VALUES (1)"); - assert_eq!(qr.infer(query), true); + assert!(qr.infer(query)); assert_eq!(qr.role(), Some(Role::Primary)); let query = simple_query("SELECT * FROM test_table"); - assert_eq!(qr.infer(query), true); + assert!(qr.infer(query)); assert_eq!(qr.role(), Some(Role::Replica)); assert!(qr.query_parser_enabled()); @@ -798,8 +787,8 @@ mod test { assert_eq!(qr.active_role, None); assert_eq!(qr.active_shard, None); - assert_eq!(qr.query_parser_enabled(), true); - assert_eq!(qr.primary_reads_enabled(), false); + assert!(qr.query_parser_enabled()); + assert!(!qr.primary_reads_enabled()); let q1 = simple_query("SET SERVER ROLE TO 'primary'"); assert!(qr.try_execute_command(q1) != None); @@ -807,7 +796,7 @@ mod test { let q2 = simple_query("SET SERVER ROLE TO 'default'"); assert!(qr.try_execute_command(q2) != None); - assert_eq!(qr.active_role.unwrap(), pool_settings.clone().default_role); + assert_eq!(qr.active_role.unwrap(), pool_settings.default_role); // Here we go :) let q3 = simple_query("SELECT * FROM test WHERE id = 5 AND values IN (1, 2, 3)"); diff --git a/src/scram.rs b/src/scram.rs index 514ed7a12..8c89f9591 100644 --- a/src/scram.rs +++ b/src/scram.rs @@ -57,7 +57,7 @@ impl ScramSha256 { /// Used for testing. pub fn from_nonce(password: &str, nonce: &str) -> ScramSha256 { - let message = BytesMut::from(&format!("{}n=,r={}", "n,,", nonce).as_bytes()[..]); + let message = BytesMut::from(format!("{}n=,r={}", "n,,", nonce).as_bytes()); ScramSha256 { password: password.to_string(), @@ -87,7 +87,7 @@ impl ScramSha256 { }; let salted_password = Self::hi( - &normalize(&self.password.as_bytes()[..]), + &normalize(self.password.as_bytes()), &salt, server_message.iterations, ); @@ -181,7 +181,7 @@ impl ScramSha256 { match hmac.verify_slice(&verifier) { Ok(_) => Ok(()), - Err(_) => return Err(Error::ServerError), + Err(_) => Err(Error::ServerError), } } @@ -220,7 +220,7 @@ impl Message { /// Parse the server SASL challenge. fn parse(message: &BytesMut) -> Result { let parts = String::from_utf8_lossy(&message[..]) - .split(",") + .split(',') .map(|s| s.to_string()) .collect::>(); @@ -268,7 +268,7 @@ mod test { #[test] fn parse_server_first_message() { let message = BytesMut::from( - &"r=fyko+d2lbbFgONRv9qkxdawL3rfcNHYJY1ZVvWVs7j,s=QSXCR+Q6sek8bf92,i=4096".as_bytes()[..], + "r=fyko+d2lbbFgONRv9qkxdawL3rfcNHYJY1ZVvWVs7j,s=QSXCR+Q6sek8bf92,i=4096".as_bytes(), ); let message = Message::parse(&message).unwrap(); assert_eq!(message.nonce, "fyko+d2lbbFgONRv9qkxdawL3rfcNHYJY1ZVvWVs7j"); @@ -279,7 +279,7 @@ mod test { #[test] fn parse_server_last_message() { let f = FinalMessage::parse(&BytesMut::from( - &"v=U+ppxD5XUKtradnv8e2MkeupiA8FU87Sg8CXzXHDAzw".as_bytes()[..], + "v=U+ppxD5XUKtradnv8e2MkeupiA8FU87Sg8CXzXHDAzw".as_bytes(), )) .unwrap(); assert_eq!( @@ -309,12 +309,12 @@ mod test { assert_eq!(std::str::from_utf8(&message).unwrap(), client_first); let result = scram - .update(&BytesMut::from(&server_first.as_bytes()[..])) + .update(&BytesMut::from(server_first.as_bytes())) .unwrap(); assert_eq!(std::str::from_utf8(&result).unwrap(), client_final); scram - .finish(&BytesMut::from(&server_final.as_bytes()[..])) + .finish(&BytesMut::from(server_final.as_bytes())) .unwrap(); } } diff --git a/src/server.rs b/src/server.rs index d191eb74d..0d8f48dff 100644 --- a/src/server.rs +++ b/src/server.rs @@ -175,7 +175,7 @@ impl Server { + sasl_response.len() as i32, // length of SASL response ); - res.put_slice(&format!("{}\0", SCRAM_SHA_256).as_bytes()[..]); + res.put_slice(format!("{}\0", SCRAM_SHA_256).as_bytes()); res.put_i32(sasl_response.len() as i32); res.put(sasl_response); @@ -315,19 +315,19 @@ impl Server { let mut server = Server { address: address.clone(), read: BufReader::new(read), - write: write, + write, buffer: BytesMut::with_capacity(8196), - server_info: server_info, - server_id: server_id, - process_id: process_id, - secret_key: secret_key, + server_info, + server_id, + process_id, + secret_key, in_transaction: false, data_available: false, bad: false, needs_cleanup: false, - client_server_map: client_server_map, + client_server_map, connected_at: chrono::offset::Utc::now().naive_utc(), - stats: stats, + stats, application_name: String::new(), last_activity: SystemTime::now(), }; @@ -371,7 +371,7 @@ impl Server { bytes.put_i32(process_id); bytes.put_i32(secret_key); - Ok(write_all(&mut stream, bytes).await?) + write_all(&mut stream, bytes).await } /// Send messages to the server from the client. @@ -616,7 +616,7 @@ impl Server { self.needs_cleanup = false; } - return Ok(()); + Ok(()) } /// A shorthand for `SET application_name = $1`. @@ -631,7 +631,7 @@ impl Server { .query(&format!("SET application_name = '{}'", name)) .await?); self.needs_cleanup = needs_cleanup_before; - return result; + result } else { Ok(()) } diff --git a/src/sharding.rs b/src/sharding.rs index c5ab45e70..18581dcf1 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -133,7 +133,7 @@ impl Sharder { #[inline] fn combine(mut a: u64, b: u64) -> u64 { a ^= b - .wrapping_add(0x49a0f4dd15e5a8e3 as u64) + .wrapping_add(0x49a0f4dd15e5a8e3_u64) .wrapping_add(a << 54) .wrapping_add(a >> 7); a @@ -141,7 +141,7 @@ impl Sharder { #[inline] fn pg_u32_hash(k: u32) -> u64 { - let mut a: u32 = 0x9e3779b9 as u32 + std::mem::size_of::() as u32 + 3923095 as u32; + let mut a: u32 = 0x9e3779b9_u32 + std::mem::size_of::() as u32 + 3923095_u32; let mut b = a; let c = a; diff --git a/src/stats.rs b/src/stats.rs index f6f812aa5..3f7e9d618 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -245,7 +245,7 @@ impl Default for Reporter { impl Reporter { /// Create a new Reporter instance. pub fn new(tx: Sender) -> Reporter { - Reporter { tx: tx } + Reporter { tx } } /// Send statistics to the task keeping track of stats. @@ -338,9 +338,9 @@ impl Reporter { let event = Event { name: EventName::ClientRegistered { client_id, - pool_name: pool_name.clone(), - username: username.clone(), - application_name: app_name.clone(), + pool_name, + username, + application_name: app_name, }, value: 1, }; @@ -582,7 +582,7 @@ impl Collector { let address_stats = address_stat_lookup .entry(server_info.address_id) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); let counter = address_stats .entry("total_query_count".to_string()) .or_insert(0); @@ -618,7 +618,7 @@ impl Collector { let address_stats = address_stat_lookup .entry(server_info.address_id) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); let counter = address_stats .entry("total_xact_count".to_string()) .or_insert(0); @@ -636,7 +636,7 @@ impl Collector { let address_stats = address_stat_lookup .entry(server_info.address_id) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); let counter = address_stats.entry("total_sent".to_string()).or_insert(0); *counter += stat.value; @@ -653,7 +653,7 @@ impl Collector { let address_stats = address_stat_lookup .entry(server_info.address_id) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); let counter = address_stats .entry("total_received".to_string()) .or_insert(0); @@ -683,7 +683,7 @@ impl Collector { let address_stats = address_stat_lookup .entry(server_info.address_id) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); let counter = address_stats .entry("total_wait_time".to_string()) .or_insert(0); @@ -694,7 +694,7 @@ impl Collector { server_info.pool_name.clone(), server_info.username.clone(), )) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); // We record max wait in microseconds, we do the pgbouncer second/microsecond split on admin let old_microseconds = @@ -750,7 +750,7 @@ impl Collector { // Update address aggregation stats let address_stats = address_stat_lookup .entry(address_id) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); let counter = address_stats.entry("total_errors".to_string()).or_insert(0); *counter += stat.value; } @@ -770,7 +770,7 @@ impl Collector { // Update address aggregation stats let address_stats = address_stat_lookup .entry(address_id) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); let counter = address_stats.entry("total_errors".to_string()).or_insert(0); *counter += stat.value; } @@ -891,7 +891,7 @@ impl Collector { } => { let pool_stats = pool_stat_lookup .entry((pool_name.clone(), username.clone())) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); // These are re-calculated every iteration of the loop, so we don't want to add values // from the last iteration. @@ -964,17 +964,17 @@ impl Collector { // Clear maxwait after reporting pool_stat_lookup .entry((pool_name.clone(), username.clone())) - .or_insert(HashMap::default()) + .or_insert_with(HashMap::default) .insert("maxwait_us".to_string(), 0); } EventName::UpdateAverages { address_id } => { let stats = address_stat_lookup .entry(address_id) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); let old_stats = address_old_stat_lookup .entry(address_id) - .or_insert(HashMap::default()); + .or_insert_with(HashMap::default); // Calculate averages for stat in &[ diff --git a/src/tls.rs b/src/tls.rs index 3bc4a6a0f..dbbfcdec0 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -30,12 +30,12 @@ impl Tls { pub fn new() -> Result { let config = get_config(); - let certs = match load_certs(&Path::new(&config.general.tls_certificate.unwrap())) { + let certs = match load_certs(Path::new(&config.general.tls_certificate.unwrap())) { Ok(certs) => certs, Err(_) => return Err(Error::TlsError), }; - let mut keys = match load_keys(&Path::new(&config.general.tls_private_key.unwrap())) { + let mut keys = match load_keys(Path::new(&config.general.tls_private_key.unwrap())) { Ok(keys) => keys, Err(_) => return Err(Error::TlsError), }; From de20ad20caeaaaf67f15ca371e311beeb3a2d626 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Nov 2022 09:55:15 -0700 Subject: [PATCH 257/551] chore(deps): bump chrono from 0.4.22 to 0.4.23 (#230) Bumps [chrono](https://github.com/chronotope/chrono) from 0.4.22 to 0.4.23. - [Release notes](https://github.com/chronotope/chrono/releases) - [Changelog](https://github.com/chronotope/chrono/blob/main/CHANGELOG.md) - [Commits](https://github.com/chronotope/chrono/compare/v0.4.22...v0.4.23) --- updated-dependencies: - dependency-name: chrono dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ccf29442..bd2445a6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,9 +114,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ "iana-time-zone", "js-sys", From 752862582c048feb7ab74544864fa5da40759787 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:26:22 -0700 Subject: [PATCH 258/551] chore(deps): bump sqlparser from 0.26.0 to 0.27.0 (#229) Bumps [sqlparser](https://github.com/sqlparser-rs/sqlparser-rs) from 0.26.0 to 0.27.0. - [Release notes](https://github.com/sqlparser-rs/sqlparser-rs/releases) - [Changelog](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/sqlparser-rs/sqlparser-rs/compare/v0.26.0...v0.27.0) --- updated-dependencies: - dependency-name: sqlparser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd2445a6d..9767005fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -922,9 +922,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86be66ea0b2b22749cfa157d16e2e84bf793e626a3375f4d378dc289fa03affb" +checksum = "aba319938d4bfe250a769ac88278b629701024fe16f34257f9563bc628081970" dependencies = [ "log", ] diff --git a/Cargo.toml b/Cargo.toml index 05a8bb28b..d9a72128f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = "0.26.0" +sqlparser = "0.27.0" log = "0.4" arc-swap = "1" env_logger = "0.9" From fc8d45a03731de4194d35e3ff7a9b6083c87d34b Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 16 Nov 2022 18:49:04 -0800 Subject: [PATCH 259/551] Default to using username when database isn't present on startup (#234) --- src/client.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/client.rs b/src/client.rs index 92e60150f..ec7ae852b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -359,17 +359,17 @@ where let stats = get_reporter(); let parameters = parse_startup(bytes.clone())?; - // These two parameters are mandatory by the protocol. - let pool_name = match parameters.get("database") { - Some(db) => db, - None => return Err(Error::ClientError), - }; - + // This parameter is mandatory by the protocol. let username = match parameters.get("user") { Some(user) => user, None => return Err(Error::ClientError), }; + let pool_name = match parameters.get("database") { + Some(db) => db, + None => username, + }; + let application_name = match parameters.get("application_name") { Some(application_name) => application_name, None => "pgcat", From 3322d99851a5c3dd1d4acc23e435d9a5bd402aae Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 16 Nov 2022 18:51:15 -0800 Subject: [PATCH 260/551] Adds health check setting to pool and avoids get_config in hotpath (#235) * Adds healthcheck settings to pool * fmt * Fix test --- src/pool.rs | 21 ++++++++++++++------- src/query_router.rs | 2 ++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/pool.rs b/src/pool.rs index 07b7a40bd..98c128afd 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -12,7 +12,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Instant; -use crate::config::{get_config, Address, PoolMode, Role, User}; +use crate::config::{get_config, Address, General, PoolMode, Role, User}; use crate::errors::Error; use crate::server::Server; @@ -82,6 +82,12 @@ pub struct PoolSettings { // Sharding key pub automatic_sharding_key: Option, + + // Health check timeout + pub healthcheck_timeout: u64, + + // Health check delay + pub healthcheck_delay: u64, } impl Default for PoolSettings { @@ -95,6 +101,8 @@ impl Default for PoolSettings { primary_reads_enabled: true, sharding_function: ShardingFunction::PgBigintHash, automatic_sharding_key: None, + healthcheck_delay: General::default_healthcheck_delay(), + healthcheck_timeout: General::default_healthcheck_timeout(), } } } @@ -256,6 +264,8 @@ impl ConnectionPool { primary_reads_enabled: pool_config.primary_reads_enabled, sharding_function: pool_config.sharding_function, automatic_sharding_key: pool_config.automatic_sharding_key.clone(), + healthcheck_delay: config.general.healthcheck_delay, + healthcheck_timeout: config.general.healthcheck_timeout, }, }; @@ -343,9 +353,6 @@ impl ConnectionPool { // Random load balancing candidates.shuffle(&mut thread_rng()); - let healthcheck_timeout = get_config().general.healthcheck_timeout; - let healthcheck_delay = get_config().general.healthcheck_delay as u128; - while !candidates.is_empty() { // Get the next candidate let address = match candidates.pop() { @@ -380,8 +387,8 @@ impl ConnectionPool { let server = &mut *conn; // Will return error if timestamp is greater than current system time, which it should never be set to - let require_healthcheck = - server.last_activity().elapsed().unwrap().as_millis() > healthcheck_delay; + let require_healthcheck = server.last_activity().elapsed().unwrap().as_millis() + > self.settings.healthcheck_delay as u128; // Do not issue a health check unless it's been a little while // since we last checked the server is ok. @@ -398,7 +405,7 @@ impl ConnectionPool { self.stats.server_tested(server.server_id()); match tokio::time::timeout( - tokio::time::Duration::from_millis(healthcheck_timeout), + tokio::time::Duration::from_millis(self.settings.healthcheck_timeout), server.query(";"), // Cheap query as it skips the query planner ) .await diff --git a/src/query_router.rs b/src/query_router.rs index 552c358c3..509057166 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -775,6 +775,8 @@ mod test { primary_reads_enabled: false, sharding_function: ShardingFunction::PgBigintHash, automatic_sharding_key: Some(String::from("id")), + healthcheck_delay: PoolSettings::default().healthcheck_delay, + healthcheck_timeout: PoolSettings::default().healthcheck_timeout, }; let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); From d54dfdd117e8fa682100e0bdb4376bbaa5ac46e6 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 16 Nov 2022 22:15:47 -0800 Subject: [PATCH 261/551] Adds configuration for logging connections and removes get_config from entrypoint (#236) * Adds configuration for logging connections and removes get_config from entrypoint * typo * rename connection config var and add to toml files * update config log * fmt --- .circleci/pgcat.toml | 6 ++++++ examples/docker/pgcat.toml | 6 ++++++ pgcat.toml | 6 ++++++ src/client.rs | 24 ++++++++++++++++++------ src/config.rs | 16 ++++++++++++++++ src/main.rs | 24 ++++++++++++++++++------ 6 files changed, 70 insertions(+), 12 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 56aa1ddc2..2a959d590 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -32,6 +32,12 @@ shutdown_timeout = 5000 # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # Seconds +# If we should log client connections +log_client_connections = false + +# If we should log client disconnections +log_client_disconnections = false + # Reload config automatically if it changes. autoreload = true diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index 97f1c9c35..c41c8cdd6 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -32,6 +32,12 @@ shutdown_timeout = 60000 # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # seconds +# If we should log client connections +log_client_connections = false + +# If we should log client disconnections +log_client_disconnections = false + # Reload config automatically if it changes. autoreload = false diff --git a/pgcat.toml b/pgcat.toml index 8d588043d..69eec394c 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -32,6 +32,12 @@ shutdown_timeout = 60000 # For how long to ban a server if it fails a health check (seconds). ban_time = 60 # seconds +# If we should log client connections +log_client_connections = false + +# If we should log client disconnections +log_client_disconnections = false + # Reload config automatically if it changes. autoreload = false diff --git a/src/client.rs b/src/client.rs index ec7ae852b..56eac274f 100644 --- a/src/client.rs +++ b/src/client.rs @@ -98,6 +98,8 @@ pub async fn client_entrypoint( shutdown: Receiver<()>, drain: Sender, admin_only: bool, + tls_certificate: Option, + log_client_connections: bool, ) -> Result<(), Error> { // Figure out if the client wants TLS or not. let addr = stream.peer_addr().unwrap(); @@ -105,10 +107,8 @@ pub async fn client_entrypoint( match get_startup::(&mut stream).await { // Client requested a TLS connection. Ok((ClientConnectionType::Tls, _)) => { - let config = get_config(); - // TLS settings are configured, will setup TLS now. - if config.general.tls_certificate != None { + if tls_certificate != None { debug!("Accepting TLS request"); let mut yes = BytesMut::new(); @@ -118,7 +118,11 @@ pub async fn client_entrypoint( // Negotiate TLS. match startup_tls(stream, client_server_map, shutdown, admin_only).await { Ok(mut client) => { - info!("Client {:?} connected (TLS)", addr); + if log_client_connections { + info!("Client {:?} connected (TLS)", addr); + } else { + debug!("Client {:?} connected (TLS)", addr); + } if !client.is_admin() { let _ = drain.send(1).await; @@ -162,7 +166,11 @@ pub async fn client_entrypoint( .await { Ok(mut client) => { - info!("Client {:?} connected (plain)", addr); + if log_client_connections { + info!("Client {:?} connected (plain)", addr); + } else { + debug!("Client {:?} connected (plain)", addr); + } if !client.is_admin() { let _ = drain.send(1).await; @@ -203,7 +211,11 @@ pub async fn client_entrypoint( .await { Ok(mut client) => { - info!("Client {:?} connected (plain)", addr); + if log_client_connections { + info!("Client {:?} connected (plain)", addr); + } else { + debug!("Client {:?} connected (plain)", addr); + } if !client.is_admin() { let _ = drain.send(1).await; diff --git a/src/config.rs b/src/config.rs index 8e1e14eeb..39bff1bea 100644 --- a/src/config.rs +++ b/src/config.rs @@ -157,6 +157,12 @@ pub struct General { #[serde(default = "General::default_connect_timeout")] pub connect_timeout: u64, + #[serde(default)] // False + pub log_client_connections: bool, + + #[serde(default)] // False + pub log_client_disconnections: bool, + #[serde(default = "General::default_shutdown_timeout")] pub shutdown_timeout: u64, @@ -220,6 +226,8 @@ impl Default for General { healthcheck_timeout: Self::default_healthcheck_timeout(), healthcheck_delay: Self::default_healthcheck_delay(), ban_time: Self::default_ban_time(), + log_client_connections: false, + log_client_disconnections: false, autoreload: false, tls_certificate: None, tls_private_key: None, @@ -517,6 +525,14 @@ impl Config { self.general.healthcheck_timeout ); info!("Connection timeout: {}ms", self.general.connect_timeout); + info!( + "Log client connections: {}", + self.general.log_client_connections + ); + info!( + "Log client disconnections: {}", + self.general.log_client_disconnections + ); info!("Shutdown timeout: {}ms", self.general.shutdown_timeout); info!("Healthcheck delay: {}ms", self.general.healthcheck_delay); match self.general.tls_certificate.clone() { diff --git a/src/main.rs b/src/main.rs index 2572d9f3f..a05ce90a5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -44,7 +44,7 @@ use jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; -use log::{error, info, warn}; +use log::{debug, error, info, warn}; use parking_lot::Mutex; use pgcat::format_duration; use tokio::net::TcpListener; @@ -247,6 +247,8 @@ async fn main() { let drain_tx = drain_tx.clone(); let client_server_map = client_server_map.clone(); + let tls_certificate = config.general.tls_certificate.clone(); + tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); @@ -256,6 +258,8 @@ async fn main() { shutdown_rx, drain_tx, admin_only, + tls_certificate.clone(), + config.general.log_client_connections, ) .await { @@ -263,11 +267,19 @@ async fn main() { let duration = chrono::offset::Utc::now().naive_utc() - start; - info!( - "Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); + if config.general.log_client_disconnections { + info!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + } else { + debug!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + } } Err(err) => { From 19dd9defd9207aecdda082b013cc18dee3bc8b25 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 16 Nov 2022 22:16:16 -0800 Subject: [PATCH 262/551] Move ClientBadStartup error log to debug (#237) --- src/main.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index a05ce90a5..0b5f7324e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -283,7 +283,11 @@ async fn main() { } Err(err) => { - warn!("Client disconnected with error {:?}", err); + match err { + errors::Error::ClientBadStartup => debug!("Client disconnected with error {:?}", err), + _ => warn!("Client disconnected with error {:?}", err), + } + } }; }); From a9896b1ced0b1604dfe7bd6ff44cde1a5716bf90 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 17 Nov 2022 09:22:12 -0800 Subject: [PATCH 263/551] Move get_config in startup to admin branch to scope down usage (#238) --- src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index 56eac274f..7bdb4977b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -367,7 +367,6 @@ where shutdown: Receiver<()>, admin_only: bool, ) -> Result, Error> { - let config = get_config(); let stats = get_reporter(); let parameters = parse_startup(bytes.clone())?; @@ -440,6 +439,7 @@ where // Authenticate admin user. let (transaction_mode, server_info) = if admin { + let config = get_config(); // Compare server and client hashes. let password_hash = md5_hash_password( &config.general.admin_username, From 55403bb562f23072c07b7388aeddaaf282ee0c44 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 17 Nov 2022 09:24:39 -0800 Subject: [PATCH 264/551] Adds details to errors and fixes error propagation bug (#239) --- src/admin.rs | 5 ++++- src/client.rs | 46 +++++++++++++++++++++++++++++++--------------- src/errors.rs | 6 +++--- src/messages.rs | 30 ++++++++++++++++++++++++------ src/scram.rs | 12 ++++++------ src/server.rs | 43 ++++++++++++++++++++++++++----------------- 6 files changed, 94 insertions(+), 48 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index d4979fdb5..4460f9821 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -37,7 +37,10 @@ where let code = query.get_u8() as char; if code != 'Q' { - return Err(Error::ProtocolSyncError); + return Err(Error::ProtocolSyncError(format!( + "Invalid code, expected 'Q' but got '{}'", + code + ))); } let len = query.get_i32() as usize; diff --git a/src/client.rs b/src/client.rs index 7bdb4977b..b55906b2b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -189,7 +189,12 @@ pub async fn client_entrypoint( } // Client probably disconnected rejecting our plain text connection. - _ => Err(Error::ProtocolSyncError), + Ok((ClientConnectionType::Tls, _)) + | Ok((ClientConnectionType::CancelQuery, _)) => Err(Error::ProtocolSyncError( + format!("Bad postgres client (plain)"), + )), + + Err(err) => Err(err), } } } @@ -297,7 +302,10 @@ where // Something else, probably something is wrong and it's not our fault, // e.g. badly implemented Postgres client. - _ => Err(Error::ProtocolSyncError), + _ => Err(Error::ProtocolSyncError(format!( + "Unexpected startup code: {}", + code + ))), } } @@ -343,7 +351,11 @@ pub async fn startup_tls( } // Bad Postgres client. - _ => Err(Error::ProtocolSyncError), + Ok((ClientConnectionType::Tls, _)) | Ok((ClientConnectionType::CancelQuery, _)) => Err( + Error::ProtocolSyncError(format!("Bad postgres client (tls)")), + ), + + Err(err) => Err(err), } } @@ -373,7 +385,11 @@ where // This parameter is mandatory by the protocol. let username = match parameters.get("user") { Some(user) => user, - None => return Err(Error::ClientError), + None => { + return Err(Error::ClientError( + "Missing user parameter on client startup".to_string(), + )) + } }; let pool_name = match parameters.get("database") { @@ -416,25 +432,27 @@ where let code = match read.read_u8().await { Ok(p) => p, - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading password code from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))), }; // PasswordMessage if code as char != 'p' { - debug!("Expected p, got {}", code as char); - return Err(Error::ProtocolSyncError); + return Err(Error::ProtocolSyncError(format!( + "Expected p, got {}", + code as char + ))); } let len = match read.read_i32().await { Ok(len) => len, - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading password message length from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))), }; let mut password_response = vec![0u8; (len - 4) as usize]; match read.read_exact(&mut password_response).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading password message from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))), }; // Authenticate admin user. @@ -451,7 +469,7 @@ where warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name); wrong_password(&mut write, username).await?; - return Err(Error::ClientError); + return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))); } (false, generate_server_info_for_admin()) @@ -470,8 +488,7 @@ where ) .await?; - warn!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name); - return Err(Error::ClientError); + return Err(Error::ClientError(format!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))); } }; @@ -482,7 +499,7 @@ where warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name); wrong_password(&mut write, username).await?; - return Err(Error::ClientError); + return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))); } let transaction_mode = pool.settings.pool_mode == PoolMode::Transaction; @@ -669,8 +686,7 @@ where ) .await?; - warn!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", self.pool_name, self.username, self.application_name); - return Err(Error::ClientError); + return Err(Error::ClientError(format!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", self.pool_name, self.username, self.application_name))); } }; query_router.update_pool_settings(pool.settings.clone()); diff --git a/src/errors.rs b/src/errors.rs index 50301f366..7789a8a77 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -3,13 +3,13 @@ /// Various errors. #[derive(Debug, PartialEq)] pub enum Error { - SocketError, + SocketError(String), ClientBadStartup, - ProtocolSyncError, + ProtocolSyncError(String), ServerError, BadConfig, AllServersDown, - ClientError, + ClientError(String), TlsError, StatementTimeout, ShuttingDown, diff --git a/src/messages.rs b/src/messages.rs index 0d7bc5759..826508ee5 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -136,7 +136,11 @@ pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Resu match stream.write_all(&startup).await { Ok(_) => Ok(()), - Err(_) => Err(Error::SocketError), + Err(_) => { + return Err(Error::SocketError(format!( + "Error writing startup to server socket" + ))) + } } } @@ -450,7 +454,7 @@ where { match stream.write_all(&buf).await { Ok(_) => Ok(()), - Err(_) => Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error writing to socket"))), } } @@ -461,7 +465,7 @@ where { match stream.write_all(&buf).await { Ok(_) => Ok(()), - Err(_) => Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error writing to socket"))), } } @@ -472,19 +476,33 @@ where { let code = match stream.read_u8().await { Ok(code) => code, - Err(_) => return Err(Error::SocketError), + Err(_) => { + return Err(Error::SocketError(format!( + "Error reading message code from socket" + ))) + } }; let len = match stream.read_i32().await { Ok(len) => len, - Err(_) => return Err(Error::SocketError), + Err(_) => { + return Err(Error::SocketError(format!( + "Error reading message len from socket, code: {:?}", + code + ))) + } }; let mut buf = vec![0u8; len as usize - 4]; match stream.read_exact(&mut buf).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError), + Err(_) => { + return Err(Error::SocketError(format!( + "Error reading message from socket, code: {:?}", + code + ))) + } }; let mut bytes = BytesMut::with_capacity(len as usize + 1); diff --git a/src/scram.rs b/src/scram.rs index 8c89f9591..c3f920da9 100644 --- a/src/scram.rs +++ b/src/scram.rs @@ -78,12 +78,12 @@ impl ScramSha256 { let server_message = Message::parse(message)?; if !server_message.nonce.starts_with(&self.nonce) { - return Err(Error::ProtocolSyncError); + return Err(Error::ProtocolSyncError(format!("SCRAM"))); } let salt = match base64::decode(&server_message.salt) { Ok(salt) => salt, - Err(_) => return Err(Error::ProtocolSyncError), + Err(_) => return Err(Error::ProtocolSyncError(format!("SCRAM"))), }; let salted_password = Self::hi( @@ -163,7 +163,7 @@ impl ScramSha256 { let verifier = match base64::decode(&final_message.value) { Ok(verifier) => verifier, - Err(_) => return Err(Error::ProtocolSyncError), + Err(_) => return Err(Error::ProtocolSyncError(format!("SCRAM"))), }; let mut hmac = match Hmac::::new_from_slice(&self.salted_password) { @@ -225,14 +225,14 @@ impl Message { .collect::>(); if parts.len() != 3 { - return Err(Error::ProtocolSyncError); + return Err(Error::ProtocolSyncError(format!("SCRAM"))); } let nonce = str::replace(&parts[0], "r=", ""); let salt = str::replace(&parts[1], "s=", ""); let iterations = match str::replace(&parts[2], "i=", "").parse::() { Ok(iterations) => iterations, - Err(_) => return Err(Error::ProtocolSyncError), + Err(_) => return Err(Error::ProtocolSyncError(format!("SCRAM"))), }; Ok(Message { @@ -252,7 +252,7 @@ impl FinalMessage { /// Parse the server final validation message. pub fn parse(message: &BytesMut) -> Result { if !message.starts_with(b"v=") || message.len() < 4 { - return Err(Error::ProtocolSyncError); + return Err(Error::ProtocolSyncError(format!("SCRAM"))); } Ok(FinalMessage { diff --git a/src/server.rs b/src/server.rs index 0d8f48dff..65fb8d9eb 100644 --- a/src/server.rs +++ b/src/server.rs @@ -86,7 +86,10 @@ impl Server { Ok(stream) => stream, Err(err) => { error!("Could not connect to server: {}", err); - return Err(Error::SocketError); + return Err(Error::SocketError(format!( + "Could not connect to server: {}", + err + ))); } }; @@ -106,12 +109,12 @@ impl Server { loop { let code = match stream.read_u8().await { Ok(code) => code as char, - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading message code on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; let len = match stream.read_i32().await { Ok(len) => len, - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading message len on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; trace!("Message: {}", code); @@ -122,7 +125,7 @@ impl Server { // Determine which kind of authentication is required, if any. let auth_code = match stream.read_i32().await { Ok(auth_code) => auth_code, - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading auth code on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; trace!("Auth: {}", auth_code); @@ -135,7 +138,7 @@ impl Server { match stream.read_exact(&mut salt).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading salt on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; md5_password(&mut stream, &user.username, &user.password, &salt[..]) @@ -151,7 +154,7 @@ impl Server { match stream.read_exact(&mut sasl_auth).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading sasl message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; let sasl_type = String::from_utf8_lossy(&sasl_auth[..sasl_len - 2]); @@ -193,7 +196,7 @@ impl Server { match stream.read_exact(&mut sasl_data).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading sasl cont message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; let msg = BytesMut::from(&sasl_data[..]); @@ -214,7 +217,7 @@ impl Server { let mut sasl_final = vec![0u8; len as usize - 8]; match stream.read_exact(&mut sasl_final).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading sasl final message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; match scram.finish(&BytesMut::from(&sasl_final[..])) { @@ -240,7 +243,7 @@ impl Server { 'E' => { let error_code = match stream.read_u8().await { Ok(error_code) => error_code, - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading error code message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; trace!("Error: {}", error_code); @@ -256,7 +259,7 @@ impl Server { match stream.read_exact(&mut error).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading error message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; // TODO: the error message contains multiple fields; we can decode them and @@ -275,7 +278,7 @@ impl Server { match stream.read_exact(&mut param).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading parameter status message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; // Save the parameter so we can pass it to the client later. @@ -292,12 +295,12 @@ impl Server { // See: . process_id = match stream.read_i32().await { Ok(id) => id, - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading process id message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; secret_key = match stream.read_i32().await { Ok(id) => id, - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading secret key message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; } @@ -307,7 +310,7 @@ impl Server { match stream.read_exact(&mut idle).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError), + Err(_) => return Err(Error::SocketError(format!("Error reading transaction status message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; let (read, write) = stream.into_split(); @@ -341,7 +344,10 @@ impl Server { // Means we implemented the protocol wrong or we're not talking to a Postgres server. _ => { error!("Unknown code: {}", code); - return Err(Error::ProtocolSyncError); + return Err(Error::ProtocolSyncError(format!( + "Unknown server code: {}", + code + ))); } }; } @@ -359,7 +365,7 @@ impl Server { Ok(stream) => stream, Err(err) => { error!("Could not connect to server: {}", err); - return Err(Error::SocketError); + return Err(Error::SocketError(format!("Error reading cancel message"))); } }; @@ -438,7 +444,10 @@ impl Server { // Something totally unexpected, this is not a Postgres server we know. _ => { self.bad = true; - return Err(Error::ProtocolSyncError); + return Err(Error::ProtocolSyncError(format!( + "Unknown transaction state: {}", + transaction_state + ))); } }; From 6f20d225bc734e5c6e28849fc0aee713568a59db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Nov 2022 08:43:00 -0800 Subject: [PATCH 265/551] chore(deps): bump bytes from 1.2.1 to 1.3.0 (#240) Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.2.1 to 1.3.0. - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/commits) --- updated-dependencies: - dependency-name: bytes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9767005fa..db105c675 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,9 +96,9 @@ checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" [[package]] name = "bytes" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "cc" From 0806038d79778479c8f0781541a950bb6e65e7bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Nov 2022 01:27:06 -0800 Subject: [PATCH 266/551] chore(deps): bump env_logger from 0.9.3 to 0.10.0 (#241) Bumps [env_logger](https://github.com/rust-cli/env_logger) from 0.9.3 to 0.10.0. - [Release notes](https://github.com/rust-cli/env_logger/releases) - [Changelog](https://github.com/rust-cli/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-cli/env_logger/compare/v0.9.3...v0.10.0) --- updated-dependencies: - dependency-name: env_logger dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 95 ++++++++++++++++++++++++++++++++++++++++++++---------- Cargo.toml | 2 +- 2 files changed, 79 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db105c675..4c761a396 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,17 +37,6 @@ dependencies = [ "syn", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.0.1" @@ -219,17 +208,38 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" dependencies = [ - "atty", "humantime", + "is-terminal", "log", "regex", "termcolor", ] +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "exitcode" version = "1.1.2" @@ -344,6 +354,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "hmac" version = "0.12.1" @@ -451,6 +470,28 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "io-lifetimes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7d367024b3f3414d8e01f437f704f41a9f64ab36f9067fa73e526ad4c763c87" +dependencies = [ + "libc", + "windows-sys 0.42.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae5bc6e2eb41c9def29a3e0f1306382807764b9b53112030eff57435667352d" +dependencies = [ + "hermit-abi 0.2.6", + "io-lifetimes", + "rustix", + "windows-sys 0.42.0", +] + [[package]] name = "itoa" version = "1.0.3" @@ -495,9 +536,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.126" +version = "0.2.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" [[package]] name = "link-cplusplus" @@ -508,6 +549,12 @@ dependencies = [ "cc", ] +[[package]] +name = "linux-raw-sys" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" + [[package]] name = "lock_api" version = "0.4.6" @@ -578,7 +625,7 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", ] @@ -795,6 +842,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustix" +version = "0.36.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b1fbb4dfc4eb1d390c02df47760bb19a84bb80b301ecc947ab5406394d8223e" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.42.0", +] + [[package]] name = "rustls" version = "0.20.6" diff --git a/Cargo.toml b/Cargo.toml index d9a72128f..d39e7821e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ once_cell = "1" sqlparser = "0.27.0" log = "0.4" arc-swap = "1" -env_logger = "0.9" +env_logger = "0.10" parking_lot = "0.12.1" hmac = "0.12" sha2 = "0.10" From 1d48a42c9dee1f0ef22d3c3c0805384d6cb0be86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:07:57 -0800 Subject: [PATCH 267/551] chore(deps): bump serde from 1.0.147 to 1.0.148 (#242) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.147 to 1.0.148. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.147...v1.0.148) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4c761a396..5ccdd6f1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -901,9 +901,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.147" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" [[package]] name = "serde_derive" From 115541eb555f75871f3463adfee3c7e2b440768f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 12:08:07 -0800 Subject: [PATCH 268/551] chore(deps): bump serde_derive from 1.0.147 to 1.0.148 (#243) Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.147 to 1.0.148. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.147...v1.0.148) --- updated-dependencies: - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ccdd6f1b..be4fb07dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -907,9 +907,9 @@ checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" dependencies = [ "proc-macro2", "quote", @@ -1008,9 +1008,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" dependencies = [ "proc-macro2", "quote", From ba03f8e35fc5eafc3e59e6fc42e1ef34976adf30 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 13:47:42 -0800 Subject: [PATCH 269/551] chore(deps): bump sha-1 from 0.10.0 to 0.10.1 (#244) Bumps [sha-1](https://github.com/RustCrypto/hashes) from 0.10.0 to 0.10.1. - [Release notes](https://github.com/RustCrypto/hashes/releases) - [Commits](https://github.com/RustCrypto/hashes/compare/sha-1-v0.10.0...md2-v0.10.1) --- updated-dependencies: - dependency-name: sha-1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be4fb07dc..ea9cce068 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -918,9 +918,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", "cpufeatures", From a3fcad09162c61224f227b1f26ca33803a350087 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 08:58:13 -0800 Subject: [PATCH 270/551] chore(deps): bump async-trait from 0.1.58 to 0.1.59 (#245) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.58 to 0.1.59. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/compare/0.1.58...0.1.59) --- updated-dependencies: - dependency-name: async-trait dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea9cce068..d898192ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" dependencies = [ "proc-macro2", "quote", From be6ccc1c6798fc426865291f42d6bf893731459b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Dec 2022 22:56:29 -0800 Subject: [PATCH 271/551] chore(deps): bump sqlparser from 0.27.0 to 0.28.0 (#248) Bumps [sqlparser](https://github.com/sqlparser-rs/sqlparser-rs) from 0.27.0 to 0.28.0. - [Release notes](https://github.com/sqlparser-rs/sqlparser-rs/releases) - [Changelog](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/sqlparser-rs/sqlparser-rs/compare/v0.27.0...v0.28.0) --- updated-dependencies: - dependency-name: sqlparser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d898192ed..c5db3e5bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -983,9 +983,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba319938d4bfe250a769ac88278b629701024fe16f34257f9563bc628081970" +checksum = "249ae674b9f636b8ff64d8bfe218774cf05a26de40fd9f358669dccc4c0a9d7d" dependencies = [ "log", ] diff --git a/Cargo.toml b/Cargo.toml index d39e7821e..ca97c2846 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = "0.27.0" +sqlparser = "0.28.0" log = "0.4" arc-swap = "1" env_logger = "0.10" From 45b204b4978cc00bb1eea54f77a5cc3a7a623261 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Dec 2022 22:56:39 -0800 Subject: [PATCH 272/551] chore(deps): bump serde_derive from 1.0.148 to 1.0.149 (#247) Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.148 to 1.0.149. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.148...v1.0.149) --- updated-dependencies: - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c5db3e5bf..c0e6b578e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -907,9 +907,9 @@ checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" [[package]] name = "serde_derive" -version = "1.0.148" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" +checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" dependencies = [ "proc-macro2", "quote", From a11f767afd8d75c88c85dbce8bf3c4deb2a99ffb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Dec 2022 22:56:49 -0800 Subject: [PATCH 273/551] chore(deps): bump serde from 1.0.148 to 1.0.149 (#246) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.148 to 1.0.149. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.148...v1.0.149) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0e6b578e..29992a3bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -901,9 +901,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.148" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" +checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" [[package]] name = "serde_derive" From ee2335fb33b652e736b3910f1cbd18c95a07e625 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 09:58:50 -0800 Subject: [PATCH 274/551] chore(deps): bump base64 from 0.13.1 to 0.20.0 (#250) Bumps [base64](https://github.com/marshallpierce/rust-base64) from 0.13.1 to 0.20.0. - [Release notes](https://github.com/marshallpierce/rust-base64/releases) - [Changelog](https://github.com/marshallpierce/rust-base64/blob/master/RELEASE-NOTES.md) - [Commits](https://github.com/marshallpierce/rust-base64/compare/v0.13.1...v0.20.0) --- updated-dependencies: - dependency-name: base64 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 10 ++++++++-- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29992a3bc..e9f482a3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,6 +49,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "bb8" version = "0.8.0" @@ -664,7 +670,7 @@ version = "0.6.0-alpha1" dependencies = [ "arc-swap", "async-trait", - "base64", + "base64 0.20.0", "bb8", "bytes", "chrono", @@ -874,7 +880,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" dependencies = [ - "base64", + "base64 0.13.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ca97c2846..d29f4c753 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ env_logger = "0.10" parking_lot = "0.12.1" hmac = "0.12" sha2 = "0.10" -base64 = "0.13" +base64 = "0.20" stringprep = "0.1" tokio-rustls = "0.23" rustls-pemfile = "1" From 9b21b34c4ec22df3fd5f827bba4f73e0ed2fe6b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 09:59:05 -0800 Subject: [PATCH 275/551] chore(deps): bump serde_derive from 1.0.149 to 1.0.150 (#251) Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.149 to 1.0.150. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.149...v1.0.150) --- updated-dependencies: - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9f482a3c..8ad0625cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -913,9 +913,9 @@ checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" [[package]] name = "serde_derive" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" +checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" dependencies = [ "proc-macro2", "quote", From ce79462bb36de9b41e9541896505a03cb847c121 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 09:59:16 -0800 Subject: [PATCH 276/551] chore(deps): bump serde from 1.0.149 to 1.0.150 (#252) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.149 to 1.0.150. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.149...v1.0.150) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ad0625cf..a5da2d9fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -907,9 +907,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" +checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" [[package]] name = "serde_derive" From 9914423c258dcef094c1348870bef0cae869ec78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 21:24:15 -0800 Subject: [PATCH 277/551] chore(deps): bump toml from 0.5.9 to 0.5.10 (#256) Bumps [toml](https://github.com/toml-rs/toml) from 0.5.9 to 0.5.10. - [Release notes](https://github.com/toml-rs/toml/releases) - [Commits](https://github.com/toml-rs/toml/commits/toml-v0.5.10) --- updated-dependencies: - dependency-name: toml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5da2d9fb..e14cdfe4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1116,9 +1116,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" dependencies = [ "serde", ] From ae420f06bccf2aa978affbc430991d5e6d3ecb38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Fri, 16 Dec 2022 17:01:00 +0100 Subject: [PATCH 278/551] Allow setting `idle_timeout` for server connections. (#257) In postgres, you can specify an `idle_session_timeout` which will close sessions idling for that amount of time. If a session is closed because of a timeout, PgCat will erroneously mark the server as unhealthy as the next health check will return an error because the connection was drop, if no health check is to be executed, it will simply fail trying to send the query to the server for the same reason, the conn was drop. Given that bb8 allows configuring an idle_timeout for pools, it would be nice to allow setting this parameter in the config file, this way you can set it to something shorter than the server one. Also, server pool will be kept smaller in moments of less traffic. Actually, currently this value is set as its default in bb8, which is 10 minutes. This changes allows setting the parameter using the config file. It can be set both globally and per pool. When creating the pool, if the pool don't have it defined, global value is used. --- pgcat.toml | 6 ++++++ src/config.rs | 23 +++++++++++++++++++++++ src/pool.rs | 6 ++++++ 3 files changed, 35 insertions(+) diff --git a/pgcat.toml b/pgcat.toml index 69eec394c..73afc4bdf 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -20,6 +20,9 @@ prometheus_exporter_port = 9930 # How long to wait before aborting a server connection (ms). connect_timeout = 5000 +# How long an idle connection with a server is left open (ms). +idle_timeout = 30000 + # How much time to give the health check query to return with a result (ms). healthcheck_timeout = 1000 @@ -92,6 +95,9 @@ sharding_function = "pg_bigint_hash" # Automatically parse this from queries and route queries to the right shard! automatic_sharding_key = "id" +# Idle timeout can be overwritten in the pool +idle_timeout = 40000 + # Credentials for users that may connect to this cluster [pools.sharded_db.users.0] username = "sharding_user" diff --git a/src/config.rs b/src/config.rs index 39bff1bea..48bd0bbc1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -157,6 +157,9 @@ pub struct General { #[serde(default = "General::default_connect_timeout")] pub connect_timeout: u64, + #[serde(default = "General::default_idle_timeout")] + pub idle_timeout: u64, + #[serde(default)] // False pub log_client_connections: bool, @@ -197,6 +200,10 @@ impl General { 1000 } + pub fn default_idle_timeout() -> u64 { + 60000 // 10 minutes + } + pub fn default_shutdown_timeout() -> u64 { 60000 } @@ -222,6 +229,7 @@ impl Default for General { enable_prometheus_exporter: Some(false), prometheus_exporter_port: 9930, connect_timeout: General::default_connect_timeout(), + idle_timeout: General::default_idle_timeout(), shutdown_timeout: Self::default_shutdown_timeout(), healthcheck_timeout: Self::default_healthcheck_timeout(), healthcheck_delay: Self::default_healthcheck_delay(), @@ -273,6 +281,8 @@ pub struct Pool { pub connect_timeout: Option, + pub idle_timeout: Option, + pub sharding_function: ShardingFunction, #[serde(default = "Pool::default_automatic_sharding_key")] @@ -335,6 +345,7 @@ impl Default for Pool { sharding_function: ShardingFunction::PgBigintHash, automatic_sharding_key: None, connect_timeout: None, + idle_timeout: None, } } } @@ -496,6 +507,10 @@ impl From<&Config> for std::collections::HashMap { "connect_timeout".to_string(), config.general.connect_timeout.to_string(), ), + ( + "idle_timeout".to_string(), + config.general.idle_timeout.to_string(), + ), ( "healthcheck_timeout".to_string(), config.general.healthcheck_timeout.to_string(), @@ -525,6 +540,7 @@ impl Config { self.general.healthcheck_timeout ); info!("Connection timeout: {}ms", self.general.connect_timeout); + info!("Idle timeout: {}ms", self.general.idle_timeout); info!( "Log client connections: {}", self.general.log_client_connections @@ -578,6 +594,11 @@ impl Config { "[pool: {}] Connection timeout: {}ms", pool_name, connect_timeout ); + let idle_timeout = match pool_config.idle_timeout { + Some(idle_timeout) => idle_timeout, + None => self.general.idle_timeout, + }; + info!("[pool: {}] Idle timeout: {}ms", pool_name, idle_timeout); info!( "[pool: {}] Sharding function: {}", pool_name, @@ -732,8 +753,10 @@ mod test { assert_eq!(get_config().path, "pgcat.toml".to_string()); assert_eq!(get_config().general.ban_time, 60); + assert_eq!(get_config().general.idle_timeout, 30000); assert_eq!(get_config().pools.len(), 2); assert_eq!(get_config().pools["sharded_db"].shards.len(), 3); + assert_eq!(get_config().pools["sharded_db"].idle_timeout, Some(40000)); assert_eq!(get_config().pools["simple_db"].shards.len(), 1); assert_eq!(get_config().pools["sharded_db"].users.len(), 2); assert_eq!(get_config().pools["simple_db"].users.len(), 1); diff --git a/src/pool.rs b/src/pool.rs index 98c128afd..94f696271 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -224,9 +224,15 @@ impl ConnectionPool { None => config.general.connect_timeout, }; + let idle_timeout = match pool_config.idle_timeout { + Some(idle_timeout) => idle_timeout, + None => config.general.idle_timeout, + }; + let pool = Pool::builder() .max_size(user.pool_size) .connection_timeout(std::time::Duration::from_millis(connect_timeout)) + .idle_timeout(Some(std::time::Duration::from_millis(idle_timeout))) .test_on_check_out(false) .build(manager) .await From b51e38c29c9a2c183c37621a64d1e2a28a68d3c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Fri, 16 Dec 2022 20:13:13 +0100 Subject: [PATCH 279/551] Allow setting the number of runtime workers to be used. (#258) This change adds a new configuration parameter called `worker_threads` that allows setting the number of workers the Tokio Runtime will use. It defaults to 4 to maintain backward compatibility. Given that the config file parse is done asynchronously, first, a transient runtime is created for reading config, and once it has been parsed, the actual runtime that will be used for PgCat execution is created. --- pgcat.toml | 3 + src/config.rs | 8 + src/main.rs | 403 ++++++++++++++++++++++++++------------------------ 3 files changed, 221 insertions(+), 193 deletions(-) diff --git a/pgcat.toml b/pgcat.toml index 73afc4bdf..0187c16b5 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -44,6 +44,9 @@ log_client_disconnections = false # Reload config automatically if it changes. autoreload = false +# Number of worker threads the Runtime will use (4 by default). +worker_threads = 5 + # TLS # tls_certificate = "server.cert" # tls_private_key = "server.key" diff --git a/src/config.rs b/src/config.rs index 48bd0bbc1..e8be9477e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -178,6 +178,9 @@ pub struct General { #[serde(default = "General::default_ban_time")] pub ban_time: i64, + #[serde(default = "General::default_worker_threads")] + pub worker_threads: usize, + #[serde(default)] // False pub autoreload: bool, @@ -219,6 +222,10 @@ impl General { pub fn default_ban_time() -> i64 { 60 } + + pub fn default_worker_threads() -> usize { + 4 + } } impl Default for General { @@ -234,6 +241,7 @@ impl Default for General { healthcheck_timeout: Self::default_healthcheck_timeout(), healthcheck_delay: Self::default_healthcheck_delay(), ban_time: Self::default_ban_time(), + worker_threads: Self::default_worker_threads(), log_client_connections: false, log_client_disconnections: false, autoreload: false, diff --git a/src/main.rs b/src/main.rs index 0b5f7324e..aac51d67b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -49,6 +49,7 @@ use parking_lot::Mutex; use pgcat::format_duration; use tokio::net::TcpListener; use tokio::{ + runtime::Builder, signal::unix::{signal as unix_signal, SignalKind}, sync::mpsc, }; @@ -79,8 +80,7 @@ use crate::pool::{ClientServerMap, ConnectionPool}; use crate::prometheus::start_metric_server; use crate::stats::{Collector, Reporter, REPORTER}; -#[tokio::main(worker_threads = 4)] -async fn main() { +fn main() -> Result<(), Box> { env_logger::builder().format_timestamp_micros().init(); info!("Welcome to PgCat! Meow. (Version {})", VERSION); @@ -98,215 +98,232 @@ async fn main() { String::from("pgcat.toml") }; - match config::parse(&config_file).await { - Ok(_) => (), - Err(err) => { - error!("Config parse error: {:?}", err); - std::process::exit(exitcode::CONFIG); - } - }; - - let config = get_config(); + // Create a transient runtime for loading the config for the first time. + { + let runtime = Builder::new_multi_thread().worker_threads(1).build()?; - if let Some(true) = config.general.enable_prometheus_exporter { - let http_addr_str = format!( - "{}:{}", - config.general.host, config.general.prometheus_exporter_port - ); - let http_addr = match SocketAddr::from_str(&http_addr_str) { - Ok(addr) => addr, - Err(err) => { - error!("Invalid http address: {}", err); - std::process::exit(exitcode::CONFIG); - } - }; - tokio::task::spawn(async move { - start_metric_server(http_addr).await; + runtime.block_on(async { + match config::parse(&config_file).await { + Ok(_) => (), + Err(err) => { + error!("Config parse error: {:?}", err); + std::process::exit(exitcode::CONFIG); + } + }; }); } - let addr = format!("{}:{}", config.general.host, config.general.port); - - let listener = match TcpListener::bind(&addr).await { - Ok(sock) => sock, - Err(err) => { - error!("Listener socket error: {:?}", err); - std::process::exit(exitcode::CONFIG); - } - }; - - info!("Running on {}", addr); - - config.show(); - - // Tracks which client is connected to which server for query cancellation. - let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); - - // Statistics reporting. - let (stats_tx, stats_rx) = mpsc::channel(100_000); - REPORTER.store(Arc::new(Reporter::new(stats_tx.clone()))); - - // Connection pool that allows to query all shards and replicas. - match ConnectionPool::from_config(client_server_map.clone()).await { - Ok(_) => (), - Err(err) => { - error!("Pool error: {:?}", err); - std::process::exit(exitcode::CONFIG); - } - }; - - tokio::task::spawn(async move { - let mut stats_collector = Collector::new(stats_rx, stats_tx.clone()); - stats_collector.collect().await; - }); - - info!("Config autoreloader: {}", config.general.autoreload); - - let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); - let autoreload_client_server_map = client_server_map.clone(); - tokio::task::spawn(async move { - loop { - autoreload_interval.tick().await; - if config.general.autoreload { - info!("Automatically reloading config"); + let config = get_config(); - if let Ok(changed) = reload_config(autoreload_client_server_map.clone()).await { - if changed { - get_config().show() - } - }; + // Create the runtime now we know required worker_threads. + let runtime = Builder::new_multi_thread() + .worker_threads(config.general.worker_threads) + .enable_all() + .build()?; + + runtime.block_on(async move { + + if let Some(true) = config.general.enable_prometheus_exporter { + let http_addr_str = format!( + "{}:{}", + config.general.host, config.general.prometheus_exporter_port + ); + let http_addr = match SocketAddr::from_str(&http_addr_str) { + Ok(addr) => addr, + Err(err) => { + error!("Invalid http address: {}", err); + std::process::exit(exitcode::CONFIG); + } + }; + tokio::task::spawn(async move { + start_metric_server(http_addr).await; + }); + } + + let addr = format!("{}:{}", config.general.host, config.general.port); + + let listener = match TcpListener::bind(&addr).await { + Ok(sock) => sock, + Err(err) => { + error!("Listener socket error: {:?}", err); + std::process::exit(exitcode::CONFIG); } - } - }); - - let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); - let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); - let mut sighup_signal = unix_signal(SignalKind::hangup()).unwrap(); - let (shutdown_tx, _) = broadcast::channel::<()>(1); - let (drain_tx, mut drain_rx) = mpsc::channel::(2048); - let (exit_tx, mut exit_rx) = mpsc::channel::<()>(1); + }; - info!("Waiting for clients"); + info!("Running on {}", addr); - let mut admin_only = false; - let mut total_clients = 0; + config.show(); - loop { - tokio::select! { - // Reload config: - // kill -SIGHUP $(pgrep pgcat) - _ = sighup_signal.recv() => { - info!("Reloading config"); + // Tracks which client is connected to which server for query cancellation. + let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); - _ = reload_config(client_server_map.clone()).await; + // Statistics reporting. + let (stats_tx, stats_rx) = mpsc::channel(100_000); + REPORTER.store(Arc::new(Reporter::new(stats_tx.clone()))); - get_config().show(); - }, - - // Initiate graceful shutdown sequence on sig int - _ = interrupt_signal.recv() => { - info!("Got SIGINT, waiting for client connection drain now"); - admin_only = true; - - // Broadcast that client tasks need to finish - let _ = shutdown_tx.send(()); - let exit_tx = exit_tx.clone(); - let _ = drain_tx.send(0).await; - - tokio::task::spawn(async move { - let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(config.general.shutdown_timeout)); - - // First tick fires immediately. - interval.tick().await; - - // Second one in the interval time. - interval.tick().await; - - // We're done waiting. - error!("Graceful shutdown timed out. {} active clients being closed", total_clients); - - let _ = exit_tx.send(()).await; - }); - }, - - _ = term_signal.recv() => { - info!("Got SIGTERM, closing with {} clients active", total_clients); - break; - }, + // Connection pool that allows to query all shards and replicas. + match ConnectionPool::from_config(client_server_map.clone()).await { + Ok(_) => (), + Err(err) => { + error!("Pool error: {:?}", err); + std::process::exit(exitcode::CONFIG); + } + }; + + tokio::task::spawn(async move { + let mut stats_collector = Collector::new(stats_rx, stats_tx.clone()); + stats_collector.collect().await; + }); + + info!("Config autoreloader: {}", config.general.autoreload); + + let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); + let autoreload_client_server_map = client_server_map.clone(); + tokio::task::spawn(async move { + loop { + autoreload_interval.tick().await; + if config.general.autoreload { + info!("Automatically reloading config"); + + if let Ok(changed) = reload_config(autoreload_client_server_map.clone()).await { + if changed { + get_config().show() + } + }; + } + } + }); + + let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); + let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); + let mut sighup_signal = unix_signal(SignalKind::hangup()).unwrap(); + let (shutdown_tx, _) = broadcast::channel::<()>(1); + let (drain_tx, mut drain_rx) = mpsc::channel::(2048); + let (exit_tx, mut exit_rx) = mpsc::channel::<()>(1); + + info!("Waiting for clients"); + + let mut admin_only = false; + let mut total_clients = 0; + + loop { + tokio::select! { + // Reload config: + // kill -SIGHUP $(pgrep pgcat) + _ = sighup_signal.recv() => { + info!("Reloading config"); + + _ = reload_config(client_server_map.clone()).await; + + get_config().show(); + }, + + // Initiate graceful shutdown sequence on sig int + _ = interrupt_signal.recv() => { + info!("Got SIGINT, waiting for client connection drain now"); + admin_only = true; + + // Broadcast that client tasks need to finish + let _ = shutdown_tx.send(()); + let exit_tx = exit_tx.clone(); + let _ = drain_tx.send(0).await; + + tokio::task::spawn(async move { + let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(config.general.shutdown_timeout)); + + // First tick fires immediately. + interval.tick().await; + + // Second one in the interval time. + interval.tick().await; + + // We're done waiting. + error!("Graceful shutdown timed out. {} active clients being closed", total_clients); + + let _ = exit_tx.send(()).await; + }); + }, + + _ = term_signal.recv() => { + info!("Got SIGTERM, closing with {} clients active", total_clients); + break; + }, + + new_client = listener.accept() => { + let (socket, addr) = match new_client { + Ok((socket, addr)) => (socket, addr), + Err(err) => { + error!("{:?}", err); + continue; + } + }; - new_client = listener.accept() => { - let (socket, addr) = match new_client { - Ok((socket, addr)) => (socket, addr), - Err(err) => { - error!("{:?}", err); - continue; - } - }; - - let shutdown_rx = shutdown_tx.subscribe(); - let drain_tx = drain_tx.clone(); - let client_server_map = client_server_map.clone(); - - let tls_certificate = config.general.tls_certificate.clone(); - - tokio::task::spawn(async move { - let start = chrono::offset::Utc::now().naive_utc(); - - match client::client_entrypoint( - socket, - client_server_map, - shutdown_rx, - drain_tx, - admin_only, - tls_certificate.clone(), - config.general.log_client_connections, - ) - .await - { - Ok(()) => { - - let duration = chrono::offset::Utc::now().naive_utc() - start; - - if config.general.log_client_disconnections { - info!( - "Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); - } else { - debug!( - "Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); + let shutdown_rx = shutdown_tx.subscribe(); + let drain_tx = drain_tx.clone(); + let client_server_map = client_server_map.clone(); + + let tls_certificate = config.general.tls_certificate.clone(); + + tokio::task::spawn(async move { + let start = chrono::offset::Utc::now().naive_utc(); + + match client::client_entrypoint( + socket, + client_server_map, + shutdown_rx, + drain_tx, + admin_only, + tls_certificate.clone(), + config.general.log_client_connections, + ) + .await + { + Ok(()) => { + + let duration = chrono::offset::Utc::now().naive_utc() - start; + + if config.general.log_client_disconnections { + info!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + } else { + debug!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + } } - } - Err(err) => { - match err { - errors::Error::ClientBadStartup => debug!("Client disconnected with error {:?}", err), - _ => warn!("Client disconnected with error {:?}", err), + Err(err) => { + match err { + errors::Error::ClientBadStartup => debug!("Client disconnected with error {:?}", err), + _ => warn!("Client disconnected with error {:?}", err), + } + } + }; + }); + } - } - }; - }); - } + _ = exit_rx.recv() => { + break; + } - _ = exit_rx.recv() => { - break; - } + client_ping = drain_rx.recv() => { + let client_ping = client_ping.unwrap(); + total_clients += client_ping; - client_ping = drain_rx.recv() => { - let client_ping = client_ping.unwrap(); - total_clients += client_ping; - - if total_clients == 0 && admin_only { - let _ = exit_tx.send(()).await; - } + if total_clients == 0 && admin_only { + let _ = exit_tx.send(()).await; + } + } } - } - } + } - info!("Shutting down..."); + info!("Shutting down..."); + }); + Ok(()) } From db8fbf0594fced0cbe3b691cb1098101c0faf5df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 11:10:05 -0800 Subject: [PATCH 280/551] chore(deps): bump serde from 1.0.150 to 1.0.151 (#260) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.150 to 1.0.151. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.150...v1.0.151) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e14cdfe4f..cb8466bbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -907,9 +907,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" +checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" [[package]] name = "serde_derive" From 02a1060586fa05e537fa2708f0570fc51beae891 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 11:10:18 -0800 Subject: [PATCH 281/551] chore(deps): bump async-trait from 0.1.59 to 0.1.60 (#259) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.59 to 0.1.60. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/compare/0.1.59...0.1.60) --- updated-dependencies: - dependency-name: async-trait dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb8466bbb..73ebd2229 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" [[package]] name = "async-trait" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" +checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" dependencies = [ "proc-macro2", "quote", From 5c1180883fde00378a0d91089fd2c27da1e97c5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 11:10:39 -0800 Subject: [PATCH 282/551] chore(deps): bump serde_derive from 1.0.150 to 1.0.151 (#261) Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.150 to 1.0.151. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.150...v1.0.151) --- updated-dependencies: - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73ebd2229..a8830202d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -913,9 +913,9 @@ checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" [[package]] name = "serde_derive" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" +checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" dependencies = [ "proc-macro2", "quote", From 767b94b8887491924701694e2d0bdb4aeaa29d75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Dec 2022 06:01:42 -0800 Subject: [PATCH 283/551] chore(deps): bump num_cpus from 1.14.0 to 1.15.0 (#264) Bumps [num_cpus](https://github.com/seanmonstar/num_cpus) from 1.14.0 to 1.15.0. - [Release notes](https://github.com/seanmonstar/num_cpus/releases) - [Changelog](https://github.com/seanmonstar/num_cpus/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/num_cpus/compare/v1.14.0...v1.15.0) --- updated-dependencies: - dependency-name: num_cpus dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8830202d..a9663f021 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -351,15 +351,6 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -492,7 +483,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae5bc6e2eb41c9def29a3e0f1306382807764b9b53112030eff57435667352d" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "io-lifetimes", "rustix", "windows-sys 0.42.0", @@ -627,11 +618,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi 0.1.19", + "hermit-abi", "libc", ] From d893a0a47f80ffef1be9b2e7ff12301fcf9561d9 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 21 Dec 2022 09:57:53 -0500 Subject: [PATCH 284/551] Buffer copy data messages (#265) * Buffer copy data messages * Update comment --- src/server.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/server.rs b/src/server.rs index 65fb8d9eb..05a3b770e 100644 --- a/src/server.rs +++ b/src/server.rs @@ -510,9 +510,13 @@ impl Server { break; } - // CopyData: we are not buffering this one because there will be many more - // and we don't know how big this packet could be, best not to take a risk. - 'd' => break, + // CopyData + 'd' => { + // Don't flush yet, buffer until we reach limit + if self.buffer.len() >= 8196 { + break; + } + } // CopyDone // Buffer until ReadyForQuery shows up, so don't exit the loop yet. From 481f8babbfeab6c5b1dc58e75067fdaa9ae61773 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 24 Dec 2022 12:12:53 -0800 Subject: [PATCH 285/551] Remove logo, pending new logo (#267) * Remove logo, pending new logo * remove from readme --- README.md | 2 -- pgcat3.png | Bin 44624 -> 0 bytes 2 files changed, 2 deletions(-) delete mode 100644 pgcat3.png diff --git a/README.md b/README.md index 313c11008..a214377f7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -![PgCat](./pgcat3.png) - ##### PgCat: PostgreSQL at petabyte scale [![CircleCI](https://circleci.com/gh/levkk/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/levkk/pgcat/tree/main) diff --git a/pgcat3.png b/pgcat3.png deleted file mode 100644 index c24c6a112076e60338c8138dc8e9c7864a615cd1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 44624 zcmeFZWl&tr7B-5z2Y1gvaCdiicNp9qf&{k&g1fuBTX1(LSa1yv0YY+j-s5$@Kd0(` zb^n|xrZVi^y}Fn5TF-iBlPDD>X;dUaBnSuyR9P7bH3$euMBojL00&&5y}gElfFSqr z)zEQQGxZ{Oc6G9}u?Lg8`#6Ki!QM8O5D?z$)tNS~2HYuOzs+&FVRp=n(1s(|MVWhh zLy9y~7^KH)QtT5b*|!He2MCx%yna0zue_oM+E$Jg8|a@Dd;coR=|JtV{&j2B71vk# zczs(M=#LZdv*-5uOn%R8-|gqmyyTgqtBaRyrn`Gyl%DaxHWhti;WXi2d8IQ}cEX0E zLb(N6MYWGBVb3dJ1^xjTJxnY@aR>5F_i=G7Ge@^8FT$qQz8peD4|@&2?=I!;JP(ED z&Id6B$;^N1i7fs;-1Rm-Ha`rbpzTqrzhxbH@L+|yspRqhy`Vd{q_)6pd}rSBv$oM= zq_yfbsoKJrIZ7C|-S79$iObHzN$!HnT4M^#n7cX98F+NLy4we zy86wc#$p@C@hm5HUo2ymI;ww+kp&Jk%b|JM?|XJPm0EAFU~M^c8h_ucb~z&PE{A2p z)xZmUtbOTpYSXU9=#M$@t~WEZMBuIS@ROqwu{2t);;HdfNo@D(>V@G_vAhRa@-^)TInq%y z$p&)el6gF~Fcfw*#adq)R_4s)8C&OV?W$Yns-;EPALs3yx*u8Fj&{aFywHtUhMNei zGS;Tp&g>g!>Or~5T1McdB`ss~udzfwSHUZrp4)q&;&j>WH*}>rp3elSYMpbN?zfDs zTb?f@+l>#&%fhd(M7qevx{axH-E{#ogxl5kUulAOsNY#pPZ}UubkW0NH_=)3ge57Y zt55rJcTf_=OpdRijpGw2yw&OXrvE+5pRHo@Rolmn^Yt_LOKG~B+~)S=M(*uHWtD^J z)pLQOM}F`r)#k{IGSPDQ=y&DZP2<;{%Ya|C%thELSX0;F%L{=DdH<}IJxIlM2AgG1 z+x0C+eS%XXhg@Q%;MFBVf(^2L>d9o;z%BFh<{6u2s1(n|@BTXWDTLn6()jNl>Xp-B z1>$0eN|I%M5|f~HS8_PbeQH%`+ZvXO?>|pEyx6>quhA_xRZAEqnyKr#EP>GLK5OY_ zqFeRC_V=6@F;{b7kM;X9%~5__-VzjE0~YBarMzy}CrXBwWkBh&rCKIkE_QFEV)2sg zyi1zkC26jAtYhm|J^8XMbe1L?e!lE@d^RXAZ(m3a>cDcObHFykM)1m`BW#s6zE>I9 zj4+baTO1EoX2ZE6=p@!Z_^5yym!fxQQW%r`aQ+DM8Vuoj0+nR`K|gI4kvtf>KXiXPk!ZO@%1pf128nj`pVt9 za!Nwi1$~>#&qwRsVr{6roWAx)V)ym!`osB8X8G7o-*#CK=CrQ6lN(QX*SX)6N!|{d zzqYt6h@qxnDv{h%u~zVA-U&6!e}}d!eqp>ll|5(?!t2s|VR+lDxquWAcsMj~M?~}q zbK3H0rS&Aw7uUH=4dO_rn>`2Zu^f(hDM#7EG$d@yq3p zc3V+A?d73Hpr-f_Gn&wlI|TywRgfV0ux2#ej<*S^qwq)=AE8Zd#=f@<3_wWP-MBkE z3u<$c(z>$dE|YxvILDz;s~6Ot^~4E}_q8ymrdHdv(-Wkx%&#q=aT@6|)Be_Qy;CAV zGmsX?kepL!9={RZ*_mm}f~Hv4@d~YqYrDW{G)UrB<*SRKn!Hh}um>(QZS-t}5qV7$ zK~~c{QyQ7YxrTW?&nq~ndV;mmx%H`_A?%Lup{2dVLGkuzm(69i9S4%8t+RbeweP6R zl3OH>I#7x**AU_-E8hgXutvBp>+9XZO{T}dc=`;iv!9mcCd882aKS`lOWzR7lMywp z9QSftP;&}kP?$b4Sgd3*<0*&P)uWwKwn1%&YBCKu`;~_*hj`}|(hf>K(%KUG$J?qN)(}E-~ED$i0C)F0L#SkvKV!Hc!by5SZ){nWu!J{rTulc!MYNEdZGw zLP>zu#hex5K)if<7leo^SXl4r`s2m_OJkm8K0wA&|FGSr7~=jQEFEdKGWGR`^7 zn#yRaF7N5%+}u%$6r+z`o_$290=yKf`wlMycoZ3iKRLGKswsAF2<16zKGs7Jd zBG~ZxSk1aKZyH}HdQN&_++NI5(}G$4Bt%*R!tm(4UplJixrEZG(Lu6bGE~&XkJ~|g zT(?8JANEmEL*xq`P7=c4^CI}0RO!>jD9BObTSTnY&TgvgR)R(=S43-M-r5cgivPNk zX)fWVdcZLg9{q5^o(5)pBp^brCa^^3@KKb;`piwsl4-C%6HMowbNeA@%o)O@0rI6% z4b*qpY%xfa5lLKJL*q#bOVNBYi>h@*W4o7tuPj< zeux(&L;i>)Kp1t^IOY#E1X{0WMK>Yda{nHhkVDRGx#SI&Y?~(*i9Ae(ZafZxNns`4 zh&NJA>J;J2z}QMG`udsoequ^q!u{q$KgyR&UoCqG(s9)DG(8GdG16nJAJCK^Ol;pE zy_ut&(F!IIGg7UNE5V=dGQki7J29_CW2+8`q)lr)l-*~vy_66>wtbLfvcuH$0|_O2 zh#8Z4Z9vE7&~bzOU$q?N;P=m^Bot&+A77Sa+HV zUK(lPl*CI8t*`pxPL1@7zc+U=C6I^YCJ}v?{K*lsemFfK=s{g zi)KZnmNkxVs!2@u4O|4mJx7HfGL8`fYqspOr);63B7WazaM)qK|2N5-{{!jTO?Apm<| zpTcQ>N)ll!d`4kY}tBGK%UiW7~@Q;FEr_+uO1l0<`CVfn#1#1eqzS*7kS(uy!)S{q3G`4dI>{ z(S38bSH|;6-c8Y<3-}qa3g-gKw9o>m)zWkNPl{{M1!o^jv6q7yi12F>Ws4yo-brYZ zn}{a_ff5k(6r>1T6iPYm-h6~HZ~T&+lq6x}1aCFQ?vYcqvVH%gW|1zvBJ(L+N|!@> z=Rt*?J|FIhR0VxhTR;*nVLxLmq>BHr*DG3g?AtNK6mLXsD_sbbTEK_V&aBb z2sXV9mxPtA&lS})Tt$f-%=zFcgIeHKPNCz%DR$5n+vT#LQKNGr)2|@AfIUjnLBK?P z6eLyB0{x6&J#zDA_*&o+rZkV1!{Q~$N~Q~5LDFzljA{+qI0<3%fj~~@=F3gryh_l1 z14YLKO)1R8KvNRgs;gUxifTrP^1h94o;s~KC>EA9FF zmFpFb;4n>HxPd7d>9B9I{H;z@iPYdZW&Jnq1?#*=7(5xI6zYlCNEf<1qG($2MW-=> zkRUq|_T1PO!T60U7;F5naEsfPF3v7UW>u2odsz0;@B0b$Q=-$Dm9Z-Q?XYsNh*N~K zG&;1Q^P){L>I+gWisc$jyhVHUb$PQMADF)r)kDH#)Y3c9s3HDRmC z33PxJMV1I+zv4H_x8I~)yKzh6iqryfRgp8iPfy_c!P{n}tmEbd__K0J1agP$N+KR6 z*J_f`5y@B|a%ymNUm(Qe&`Qiny@vG%%v4H4gzr~1Fiz8!)=pqRkRlnIFNbzkme=&`a@Vv(CLe}<7lz;V2#@XK=;gfoLcrr<@6Kh1!ce7m?gnP$y=%g>yKDR`}E<&^nC5!N^pdgw# zH#Q`~Z}VmDblSwTj6=+)5|GlnLy5P{xEUXd^gSSrl{Rss)K7IT``U%bg3EC(oZH(~ zFX043yvp1^3Y4ga>@MfjmWg+sH54b&j{DBM!i^AkDbILgpX7&qj+Z~j=ykfLTt)6N zgciD!sd95~RkFhZX<)%1T)SUEa7Z>CZnIKu3k41fk?%+0L$MHXa+N}TS@%u}_Ms-d zJIaYWM}TDI&z^rL|DqU4J|4htUoRRkx5HvhzNsx7PY8ns)ax}HntYgsX;xp+VI;+Z8C+46CL&sBh>Ev(zcEI#Atyg?tv*a| zao5kaLqTZ|hZej(^k&))3aSu%l(H=&Xs#V2#f9M`{JoK24pdPTgoG8ZeaVA6E2lN5 zwfBV_n@m8?@(ndi$LHM|l1TASUZ{-3=h1y>MunET)?JfuANLw1jx~}PGEnC#9iRo5 zcD79-X)&DMhxiVec6C$9`k^wYJl<9-N3TeU+D@Wkn{hZqt>`8{D7B#ki6UtR$3bxO zTz7m)pelpgmv<1a#>s|GA}4UpBOH|8l_Uzap5>TMi2ckh?DfqE{!+9_u@sZkK#Blu z&CeVAM)`w?dZsjpP{8i7N=_MvBmgtxg8){v02THppD4#3bAbH&ti9fbCkDucc&%;tC{*qlRvi{%Qn`ss76}33El9g zn%IE%muV;BBiy}vT9qytPq(~+()}`J6Dm9Q@#%I1)X>!0hKr05vqX#l@Jq<6be?#B z*%(INnIbH8l5h!2k1QfmHepfT&6&iOI-5XKs7gO~R-ecs+3sR%C~sxWtV>bb3IbYF zro8?sWHu#xY})p^5!JaZCXyTIwTd-TgoV$S;Ky3iH#EEAsC&uhZryO-3i;7Np@F^U zEWRSal3^zk0cZGt*$r{cq1M(fmmVp!W1iW=17GlUrFX#~yxZ!jb|+WEg(Hl4a;Mj4 zfvbz17y(G^2nkr(%TOEx`}9<)XY>@=)Gv|I)TeoJimfIeL$b-A5{Q0n>n{rjcm`fW z2M-K6_o2S?{}~lcgdegb8f%=yA!1^;KMNYD+9TaDX_vT_BUSZ>b)K{LwHAgpSH8cN z62K@?;z#(8{1oZ{f1Zm0L9F>j97q&E8J@Z*qZld2PV+Poje3y7o9 z8W%a0ln5EZirfh3&aT2~^+D%UkFa0o$vz|oGqIh0T{nx`AGE_*8c*nyUOIy;wrXwB zZ6=!J-1Qk*j4qjLl0(Ht-I6qq$|@RvpO-urJ0>YkUC_M?M;e9M6z+gZiQ`I+_7MgW z32!p!r(Zv^^)40>b-j=a@tqz;qxPOJQ(yJUIG#b%S5fRx?sFURLrp&RbCC};MS}j{ z+OZ9b{J+GQ96e)8iBPKVXwcp9uJc_2yDN$8_HWJ#t?aI^*w_fISM{8=IC|z0^U=dhGomNmPQIb@h1r$Xs=|0vc!(_yB*i!cSB6w#C=P?|&&ML@ zT^ZVsMvHy)&Aa7C>|>ls9Y{JsU8O#~DF=%jSz|cUyp+y z*OUuV$monv_v4qY2kUa4OqiTZPk4(w^Rn=qP=Q(4O~sU`&Fm-*JL)3D+}Hq)ok&$T|ZYFAv}^k*+diw(AE@rZkxswI0llE*_DynR!f?IzuVS9BnxkXec= zVzmuUJzxn+lD+SEVTBV2Cl*axNa8Ou7eDdN*$jCPycdmqp zoiKWCIm0f3pnISFZtAKWsyyE&pA{_J3*ofJjxEDKDKt|PG;Dif&8+`bhA6J797#j$ zZQnYH96_U{DFyL^5PG{ng{uH`0%~14RA6`HqKq0BYA0P2(=~b77P=;+3FYQ zep1-qDxBTT;D(P*Ro`x&rBUOV>vOBT^Ev0QQw#!E7%-8-po&Tt3^d&#X{BC*|F?`Um zR#Hetm#elUB;756{RYOZDOtYXT##^BwFeVkNwOeyR+VsB1ko9@1V=SoAZ1fz=a@YF z4kN1Z`w3LGIyVWu=|!*0;PUGUyau#CrRvx9a=wu&bvT~n8QJN&UIt#s>P1ApmPF+F zxKOa5{zq{=SXfPzgi23g?x4QhH`*r6*BqZ6&yTS15aEP<`bK`eU2-jSv@S#UJW%Z1(jl`dVhEgN)Eeh`ya z5AXKI-aRwXJB!!7-@c~0kmo9o$zUzjKwg2-ne%5o`GHa_MS)FPp-!%W|GLS!dBzpJ zH(Vw{;_JJoy_1nl!2yyau_q?tl=zWH;XrsM`)I?Lk>`^>ZRVN1Q#Pmgq)&1;SM~0& zAdv&AnqKvjR_;0s9FmHFp_=|%*VC)$0=kE*sl0>WGo#zl{6tH>;YiXG*Aj~0-vw4w z!TQp&#~(wScB{cQc7&P6{v}VAl)M8JpKQHX%B~C+osDkK^lIHpV_gkN$Y*sYTcwfY zC{(PXc-oc}A_}}M$w-X)dgJpgaUO5Ftw8TQB16;G=a3x+64s~S_AffCQ9yA~ML6J* z({(5mjyZjTiD1>SGP-n|z~*jB+}BCSipyXSk<1u%uqZ-o$=;S?X2ADi64xPVQ>#(z z6Q*j;3}tA#xRpAaq>3!jB^)A27}H&b{o=by3+uGKT|4;r&q_w0zvjGtcZUGQjwzr| zHOISo;5#u)Ik`ARcGGaI1YDIfyqUzkK08+L54pP7^r%rSl+XFtA;TBQLb2YGb3KA- zN7#OMMr`4$B3}WUTE=-%Nj3Xc6c)kVAf-o)lu{@Di(A6yc2~T7a|>>uqz`80)#$Lqho>MvmGwPwBYUzIzyE#bQHPf72WGBc;UN_kQ2^NaE6Wc55s? zpVmLGBHDGYXLj1n*Sh)OU@9MJzp6oHr^d#9ueWx8EhUHCA}@sSs406spsN!A!J)9x zgxmI#dA6lTuT&*{J$C+99F?F{!jJ7fZA4{3C~q5h48r$X8sjC|Gte+!X&(tuiz3uM zO)6R!=TenEuZjJ@PpYp~*y`;5)yIY_k!8Xx;JWt#%4Zds$fiQ!=x%V}b>D6?@cs<0 zx9c;U7O7*j979mSBuZA_C#-n!8de4Eqeu?!GQO&OLraH%`HlBH+1L#0B0dUZ0~sI5 zqPy~;g-VB`)|6y|!}-P?ARp9|aNO5Tj#mU7Z0O3FprjT}WztB=H= zCJpd6Ks9xewi}A;**oZ_1a5%7|1Mql^(&0NYHRa1YfudxzU6X7jG;)Wr1!kHbbr&43to zB{+mEMiU;-%8<-Zs<_UO9+&c%{KX zg8)sG9x)LJ@kmN)R&`}d+{P|M8A#1JN#pQ9dY2i(&%5wJM~Tn#5YE@?Up9`zeTRrY0r?1Pw+OCaOX_A2MC z-72lKDu3l){VJmRW6s%!AS@j*EV{_gs?xBUI*Zz{gmkp?O=P!MaXi1U+bvu*zl_#@ zAk>hFCHsjU;AMwfTDnoBf`ZVs`Y?izu>q@x`t_Gk%f&L{P9R)OQ*7@t%%PC1lY7%B zCQBWvfq@r?4|>}MWO%Oo*k|G~Wpxm4&?EbEi2#c?@?CX%ck`pmGlWHXO;p!=ljn0FOLeHy=0cet_2&ro%g`QBKS7ILm@hf3Vh~H4os#NPG(>xZwF`Kp%DlO z0by@vQ*%48JGmLy%EnQU^0Kp+lHA5ZkW!mdkwwv29BgeP1P0gG<+yyBqf&1kD*q?*5qT)Zr zJG%Yd3V=PBy-l5&S(#Xv9UPedeTJL6q$eQc?;H9*&T!KJo~>b41G_nSxSE3{J;9Fd zRR1o*!u+4}ojqLb|5C@ooEdBnb^wOD0jsk9$CA>riYot{@y7&KHV)2z%>s=5A0ypu zEdOP!|Jd7~JAbM3?;8Q8|5NUNjQ)r2e+>pkDJt?wIGKC=*`BO~AmyL+`7E5wZ7lfy z`e@E=$;%67<7Q;xvfyFlu;AcjH05PuXSC#HF=H`j=V3QB2mhNWSw}Z_Q%7^~A5nmC zCL2JGIoQ&YmBo^a5zN7B!N_6B$;QabW)5cLVc}$D9-l zU}^6OJD+U25cNHoV?urbxXs>0j%zB`p2HE zTpS#%EUfIjyxg2z94zer8l(kwbpvAYk5pC`CicHc{^J-vKpnuerhg(85b)OwKpQ@B zSFowOldFc4lf5A2pG}efnfX_MlMDRKEiyK4zzCl|G5`0dR|mWN?YF-j0(+akZjqD! z6}Wt+=6_qn&D0ZY@z+AYyuaNtw>EXO0s|-f?;-V%>o)%fcbS@US(vi3a4@o)vI8Dx zWe04`!fM9I%gVyb&ce>d$-`s$pVHl&EZx0KUBRMOfTn;pKnVS%4LRuV#-#ht@m|*8 zKbB$v&NmD2!>Pf_!N|o{b`3}_K(1o` zXR`Xcz5bZ+|KjWKe)zw*0wDT-i~Lvi{hzu1XRiOs0{_+E|3%mT%=KSc;J+IDzv%k^ z%v?zSYInen00Qy?nwgL=(oCQcf-{qsmiVhBf_M_|3*m0mW4%|p_JK!SI{ zgb{(D7LBG{Q>@olllSaga#U9C?B28C?=l)0R){b+r|Pncpt7%hyc(|Z8rgGRWo@zQ zU~H9cS!!4;SS%PU@qRNXN<8Sl64r4^71tWLgCTQ`i~f01SppVwNQ~n4;QjXF%j?$V zD`{TA|L?B`bqd=z8^TFy2@?maGSvOpNU#MUB=4MvK9fAIOhs0F6eI{agnU~&!Mhf# zhBbyaF+*t*lzoGHF0`o=m?3>4f~0Y3hK7vkQ*fI*6ds8lI1j0 z3Z z!$iaR^wI57yTNq)n8o>!nQH06i5w0Nj%euuuaT}Uua>4J&K{4F!)*jaDOX>2;+VzV z?d|PIjdAaiI2Cdv0$4^n>g@%ixsu(%ws@? z7LjddXLsl)RGUaG`*wnD!X?X$E7R@b;^JL*clU3GiJQQZB&JH-FnX-G<~=24Wo}e- z^xwnpFS{Sd+ue?ytU039%A0u8@>b4FxiU>zj!iiTaazAJa&mGWoSvQrGo{+~+cq0{ z@MTji0I#K`B~ww18+3IRP%#R`)ahvjZRV6MjwGgq?(%p;Rj1!mL%s(6iG->JU9rWU zMgc=9`+Pg3e8x-EkZ_VM5~127o8X4@PMQ53<8Kv&$cw50hmIvUAL zjCQF+o7g>PU1jIvGsZ0}?4)a8u)lW=lQ{v7m!Vdr$7v-POj@$$(8{gddanJY*^zMc z#3_pL3r5N8pm#YfNHRh<7cZY6(CF$s?L8@?!6(td@TVuk3 zWFv}EK$}Qs&p`++Rp>GVsNyeV3?uR^3KOR$hb>vJtgO6JTT|m1A(r2x0$L6Z;EjF5xx2e!>(BI?#k{tN^^7ke`2?@z>b8YRFiuL!J^1Jma zAnZ144n3SGiK-_*G=bXgY@eQ`Aa2Pb4f<~z?QZ+PTg<`1!A@Fwqz2HEd1ZN-NkK^o zjf;~r8_*L^k#?cKJ8~#|XeT2l$1xBH5WfCz&hI%09U_O!zFV-_OdYtEnAMj}S+Hej z&`U-SQL?2o>|Wz1jO`0W_)nls*1n$)Z2 zk_P@VW%KO5g_DD$i$h3AXnA4bR+TcIr+mp44;3z??UjH2gu@hHN}oQNFd{rWHulpe zlAQE(8$&}wkJ{?$^u6mgu5^Wld=(&d_IujF9fPLdy4GT&lOS;0%!{wA^ z+~2XY8;gRbN_x zqsz_E*HAU-vVoV8j2{j=(>Qu`i%-xlWoc;{YPV9egpP*h8xaxlNXFy5)0~o$LN+rw znZ=f_P{}Zn5;Iib;^E<;O&Ra);pg|WI4CHnq`jS2o*ZN6O%!$Fl%*;IZfjNAOS;b` z$0`Da3I1Wq@7@+%QJyZia8r(8a-B)b#`cDWg5|UO2M^OA%kpZq(!{-7GdJ24$1QV7)<4&$qM>{+J=Gxjz$-=p`%(ONm zmtQ}96e%&^M&DQuGZkagCaTtdy@2f8%+Y>d1GpSj6azbR3Kt88VsUl#7)XL02dBdW zc`6_?YwI)uVq&hddp=jain>}V6=CWO|R-cDc< z5)w)@t_kdUb|Tx(yza69^|jGQ@!r>e*0Pb2lZ#7DH-+>f#1y}dA;h&EC3Y`|EvW5U z2*I?D$vw_W$=1eZ3?C&Nv2>w4IuzI!UgZ*@8xJwNQYrh03t_XUCw>FNz(tLeqT*D; zlw~F~c|~;rV720o4!$F2o*Mq_>9mo(zA1tjdSyLo>hrHF(hgi->ju7ERf@_3Jpd(#>)X{*+*l%qC? zf2Kx!|M)#@;vl2CtPH>NiZ9H{-Nt~#+ z(F-S8r|0Kg=eJKtdW^MPqi@2&1dEG{GD5jd<43osSYpw(m1UIIMb%LT^70cAz^+hN z?LC(cy!+D>Xn*+iXsD|%goTHP^EOLr#IIteO&tbpZEd*$G$gfLhkJP#_>beBwY2jZCNu*ybG<1$A*tG(8qSQsEqab$R*I^;Y+L zywkI@@{FJ-?>SqayT`|)&d$z2Gq!ZNC0t%tzY3E&ev`iUSA$@@WVBU4p{R+ zNP2pDNq9!yru_%5uZc>f>d0F{lsT2vyX^+C&cxDOgYg^I?P`)KV0wgp!Axkr?ASxs zc58FpAoFrW)b-zUj#{MK#>*?X~x_7-d;vW+u4;1g;qx6z=(NMZ6 z+${->(NEO?y-2m4@_Yky6@2|aR%~6hw$5fQ9 zWF>_WnzUp)g^vPP5k1H*MU7>O6(ylgj`3sq^widi0r$2-&V%V z5%ARaxC&5i&~(`dKheYtWz_3cUq>SyWz8JD0M_b3B(IJhY=qBG%5ZHrxVX7-)dXR? zQYXsl1E3VBb*${{-LH+wp>^6Wo`_#>+eA_k2=1DNUAb#%1*sL8qapsCc(pZ!V{-j9y(^D=jGr zivttHf-*4Pz+ly=OBf|F`tzp%96bD31X;l7kJo@3mgrkGIa3w{o`9d;e%n3%!hm+U zJjAi{in*ra2d+c=`xXOZiEsHS<7K3&v22t|7f{PJD(Eo~(hiUC(hd&5Rn=im-uLT=orRN0JY0mh(u`lb%)bi!b_WqV>{gg~h|vmG&In@nxTbud zK8!bfG^>|+69to#r6fN1*%tPfxY=g!SRYYre1 z;7|tapYJqOvGtfzMaV^J4cf(ieY<-&}cleilLs6Z=hx(4RGCLGMJY}Sn#cZVDl5oJ-&T9HvWBY>;^Om z`!NhNeA#GSzn;&=TYFy}+AAw#(xxm^z-#(O@B=Ior3*lp%F31b$+z36y}VquqUNCv z0H3%3`tA8lw12yF03d(Z#IYtB1l92awnPP5l9gIxVIeNAhg)%R@ou2&`<|SPBhG?v zp{4zrrvW))lechUTAv2=%*oHs&#plK($Lcrux!tTetiF|DvI$3^UKt(ytU`jZvlF$ zj}d-J`9K9)v(U?Rkt3?rDA&+aq{b>N00m@CSu%!+<(Jgf*1p9?>Sp2PU1R0uJ~GtN zVVRnqcA2vdwP@4@nv{gKHQvq9QO=p!+0S;~-rkqr1|pQ3nwowN06l61gcd5CxJvEH z2vGK+c7jTjbW?zg9q^pl`;<8!_2zBlrlS?N<=Ojo<$@(!Rek-+r(y0b6%@FTA4h?| zjxs8>;mNEm*6p>ru7`=nYk!^Aj(2)pS~lxl2xbEPI#%Y?FbR*d2qsK^)zZyf@2?dy z=K_#Wok|^pJXn{BKAN(0!DjF0&!1aB>|N#v2mUGsFlxJ%)lrk3ohc9!qI6$w1wKka z3gs%>SX+Y@S5~Bf(`w}}w2f0Z=N)wPWYf4tyaZ(PoFmE3&6+Zt>Kyg>HiBv`c}N+E z28f0sGuAY@;&7mY3jEsr{;@^{)D-CAQXlB!<8!&Vm~^z>Y}aAJGFqZ$BT=Y|6*JV9 zo=$KyICx(ef!;g2wXopU)ZBc}m;$DVE+CJdhhF0!_3K$VbJb%n^Hm%p=<)5w3L()J_*U|@=U-(V{A|mWcM~1d2~Y$ z#*m9ZMFstFjbZ2D^z^iZzyIs~`ue&&(4zp|*#Kwk-ZC+obkvcLO(Anx8KsLA*YaQt zMD&_Hmn8=wyw}D0y%q@D_G_!tcKpG?0W&HpYO_(N=Tuiy(;(29B`B0QKiz<*=I0CE z1Kmz0kl$bYgp7I86`rx82K~{?6jOJ;ZvSfW`}xCV*{r^9A9?jK6oV&Q=^WXooVNP~7O-P71`e$V^7GdET zprzj{RnGlRfRAs@@AbXQ+|@NQp|XzQ9j2H?>>jo# z26aeZ9zw7RWoT83T(JmH^uvHgnasn>i!3)Q%kSxQsWKbrreoXNeS|O&KAW(}P{x;e zy1Lq?e!Z~$C{?IHjn(axsE0k_eUPlZZ$1(`H$E=+-N^7`r!KZpZL0r8bM08J-wi9j zprD(wd;*YFq+^LyfiqraR2v9FM7L~AKRG#3MI+%WQczT^AKHD@Ok1$#NXy9?-hSQO zzMu^?2?z-KG_t3H4L0xe&Tg>NcJsX4`aT38Ugd71%TC%vn^niw5m-@>(;YGpV(9?u z)1m@$2GmWUzD=NZCDKAz!_XOb$?CuDLdXLs6QH%EVMV#!Kty@`hzOmkK&$xf-MiRE zs~N501snZx4OakTG%`%^$(55&G>9fp=ctXdpisFHYg^#cN+XP&03_d>^~baO=QeVL zdf??|T4zI62kcT#t~k12F3m48jysVT!@WZyR@PNu`!OKHm`M=jU$(N-jH4Dsw&E-Mfxl z+nIo!E)Kt1o7bmJT|Q=JIeiyK2@%a$H&*~r_}(13OA+kJ9W+=lx2F*0Q5@#Snfi+7=;rDd4_EkgfIR48-e z*s6SS0!{7=@_}nQj%55h+!%mNp#;dWJs#_3JxKs85`K~bT7H@kQJJ&*j!<$0JKPwL zA;(t6b^Df+v9U2;hzYGq4^NBGZP z?L9p?N=i!c@Lgmw2?&jAAK$!HiUb&%Mn*H>fp zv)FMXL|A}=*-Gyl*X%X(RqBphdDZh(cG%ND(k9DoIx_(b9R>`fR;4BuOuju=<^q5P zrp)TAc(SKY#>dC&=jP_l05FJ*GEgNr!O59x8;>D5clh}7*z^0>3-jd6%wtVWO$!VZ zq!$RA?ddhZG9Ummtl#kQl?8&!ZuSIn`1E#Zxx;`7NGWrcjR#puR*M&&%*a2i-3768>_N|mxt z^>uWP0gy5kFH^*a6}7HbUW5@Pu_{6i(YqZeY#s`2a`of~(1+_vtzYkfvx(!;xj6x# z1arWBsflCz0Hk8OW$2((u9krb>Dywrn*Mn+J3IRgV7S)-5TVY1yMP@dSukg95i>-2 z|M0yS0$^}J$>RXC300h=A{Rw1TV%kLs8$~8&KF`<-{ztA0O*H{mkOcssgGQzvVy)^ zskCux@0u9cM{R3Mi;|bSd)BqL09o7ScbkZTYi)qA&ga79h*6?@ku%0|R^K=jL2i4)kJ%`ZY8(J{KYfGU5WhR1hMKV+Z)f16RZ^zPP7{i1q?A z0~Tz-M^DwSw4p3XrlXdDc|@cL5aifMW)6&1nc3O*devH9nwpwN{r&xaES+imdaC{V z@zV}K5Ap{o(3&%+PGkUHw5eR;Tlo4_o;z5v0N~izQPinJN0ZdT)yCx$ihwbEx+%K4 zDfeMweHjY0Q)~8GGSmsc*9TT|#cJySiiLxMOqe>11f@_!Qvz@gVC5Qtln7h7WL8#r z>!YWa0T3LUFSo!6DHR}{c@I!EE5v9czh&vBfNvxMU;nfTZ0YCznRAaFM^v~mF>jS~ z_9ay%tgK1_g5kThB8FJroORilMMr76Qsu!(cVsfd1i;0O#0A_@nF07#Ts*{#gYXh0 zMm)kQFk|iyefWwfp{R&z^!}oG4d~%s$$)G_h8rWfH##~>gp7<_i;j-|_%SX{dunFJ z4rrSJQm~Z)C!7KUhN?;n8DKo}0izX=Bfw@WrMBuY_*84@o9Jro@+i$&a}-Dw(ls)4 z5|nNDvZ6FTCFrK$=%yrBY1u6wxGF+<)3USW>c#K){Hg->3(yfQmQA0!k%03QW{Tfo z^Gp{l1}d%sMZ-YMo!xfFUP-9OBAFNJvPE;CnuOpwRZrudm;T+S#2&{Bbp4S=ERE8`)C! zN8;R7l<}y7w&vwPt9XURdYzBD`ugHL#3=xkb9ua5bvc^^P*QIndB~#*K-gZgo2g)3 zJwrnp0CiY^!LVz<|2Xt%So&rXM{*cA^gtVNfRj4W&BM+8*qol84wONn%|HM*0nyb( zOKe1yKplhum2>pXnWqeZ%mBz91o}gJU{7Bx8>e&`QhYi$dn`Bz2l%p?#vAMt)bwaW zK}l+|s@Sr+ffz`y0?`GaXod--VhzAF(Mz_BSYi?YVTB=-TMMxhr2MHQ2}@G-vr?%n zWxPq1*1Jxt8PX7jiDhpAqj&|{GawY-({eG-QCg+(%m8Ix1Mk2o#BTv<8hxIYh8KoHuUNYaq=YTcqiTo>3FgFm5)>e zGh*bzb;}IEE6ksvQN!A+B#)aqsA+ALKHsYarkt2kB5mP#R*-zM3BVHZ(v(tcX;5Ie z*)Huj{V8;4VZcQ}kVw7|To_^zGsT;AJ6~J==d7=v>+R;&>|kgZ5IMAXqkQW@M2L<7 zw8&V#>|+rb85u8dN+acqZ%Rju@!2va*wI5NzSBs9KjEMvzT(W7h?p^lpSroV&Hu}{ zug=%?aaF)`3Cl0w?sN>|2k#f`(Bj3JCEt}I2}K1DT!n!Bj+hxK=8_;ccaxc|ZKsdB zd&v&K{m*-Q*le7f0}PCeX^5{w?}2#ZYRvYYi&$Z85e}h`MFZR;3^)aQ4(|g<*OZ_R zEqshv5vuV}Wg4vWR_+2-I$D1rKLEnnTtB|`-qn)iBmU0}JIt~bOEw{oAeSnh3me*{ z!9qb1(EE9SxO2sW7A#`S4tu<@8aB||3-|lM_uoUukyA)oM#e5A93D&Df8J79MkaC% z@U*KZub#odL9mw(!U8on&LCAPVI;A7*;*XPP<0GtRLUjf_UiNj2;JZW%h6!%%pGw! z)`bfS2%w7~6($BK;{yn{2FP;_*gPncq=~Y!v;U&>u~b-Dy|od@`BbCP|MMpnJ11wM zckH}Ohyf-N{P6571gPLZ4t6EXjL2Vel#`RD1-}4JnF-gwh4m`R|3DoFH}~8}1aj_( zElj0E(g<>6&CXp2a7rRJFNVMefWKDl8Ak6`*VoH}jptGB#OjYX&fVSJD3B260H1iJ zTlG+^RdFv3cr363-9^FxZUWAoIWS2HqaRrq41C4p!n7tP znl9i}d&p<;swGGl>VOqki3ZCWn8}`XlGaW_KdwDz{+9)axJ$?Cel}(NQliQhnQxu~ z9Z8%p{TfEX_w8rM_O<~83H}$*4fo|srZZBSl%&EpC_9DB%*>8TREnED?Kv03dli5M zv;6n3&%luro9jEW5QA>}^fnYlwlq~LDymd)Mqf{MHMeBZA^^%Sl85PeG+THKXsb6s zendYl8!SX5{V~by390P~)eZ5A#E{-{zluTalMKR)J9rBCTqfspJM-xX+$%wgCsEoJ z3p8ZV0-)S6qo=#^ZMcCC3uI?!_x2acbuQ=(0x%Oi)A66N#9jC$nskHDT)Y#MN3kL) zxw7N|hr=*hb?_CJx^TS~Bgo9#yrwS10t><4 z!-AO9o*puXs>KX>Gdtj->>th)D1+@8;EL2$%agfiezLpLuI1sOm=v}{@}>D$gl58X`;*M9QS?F4H2 z%YMdJ2LL`U+C3kCmO(9E6U0!s1lAo!};6s zlxOm}$9DobLWc#<r`3KT8nxI7(^zRr?wk*OEOmTC7LO4q zp(WsnxPQJo@x&nFt*`3n_#>DiT__B&g0lDytr(U#6DCqUAbUO!*L!1n|8D0LtjtVJ zA3>@h`ylDrgM>3p7P#Ar<)6#S=Z>It%Q~|scL-paV?z~vAF_=XGBYtgt}&JQp#a#= zzoar-ZZu<)Nyk#`-CSI0@+GhE1z2XLa0#iKcAt_gQB@{A-*X{S9z(RaA6zy0K=Qp4)b?7@)UBKuNW%(db1C?5UbN>LnZsrYWS&K5*ol ziqxuDfX)4wCVL(|w7cQP$8^b8wVoaSyO}hIlvrUa=kHO@k_Ll(=ij@W6bupp1|U~E z;75gEiiC|A?{orbW;6)X#gQJIrHFhV9ERDZoGhl;{Jz{^<8b zR}2vIATdK%-xY*8GA4eMl_j`d*iM}RNe;08GXOXD1cB?rD$KGkT}TeHP)j0t-ED$31~!L|re?|5!OcC`6vOZD@BfXRxx@hSuZbOA3?r^2NtI9%3mqA< z*8!IAYfgMQOHa;|3{%OzrKP1jphEewb8y^V|GvnnkBN$cNOP;ByU5>I|FZ1fw##j5 z`gDGDL<VJ_OfGyRr#xEz-t{?g1=hGmY7f_< z7R-cocIp;ZRVi4So0A5R+67^fkFf#R1H2~|7CY>M0wbCfnV8eqZ06ZyKxf^a_EI84 zF-SUBfYaIpgbBft@^aKQrh<6oQJCpuX-YG;wDOn%LoQ)V2|zPz*KO6oW-*wXcnwrK zW<6jT{swZ}NM6&%^Uy@|fIO~<tK>b~jO>m-u_XE(neD4*h(q>Z(@D+9_{5ab zadFY|WpUP=BW91I1{}mtG?7X)iC|~@*Ue567gyJRHfn0~e5zbAoS6|#bKJ8w}Kh!8iqf^Vb2PG%t0F_gW{OzwBcYw7eE$o8L-LTkg=9 z)QEvhE?GF+I50544}8xBF>)!FA|dNe+du!BZ~~?*m1rX1LNuoaf4S=dTK-u+lSdc` zxRL+@VTqHiTB*}T1Q^zY$x;Q1k^76}`4tsWW%Ag=;Y1>+SLz7UU~LSu=56tygOdb) z1vZz?j#XKzsD_5&&y%C_`j(dImwXp4yjfp07?SA|XA_PEq7PJ{j}M#wq(V1ImJ%eP z^?MHP2U7hZLTmrVQ7_v=KC7xK7SCy(o}V)T#TMj!ETh(KsflAk7fuo8nyjkD|cFR7{1fG)3 zf%8U#)4EGdeqw$>!K*@LF_-3nZ!0>7jsB;bTqpUF>g^zU^kW$J}fYj_ichtYKf}fm{@)kW5 zIB>OUfe8k9)>QN@>$XlVF4EvpLkBAb8^H58DPwA5V7p5by<#y$z>nI`UPF;yyb#U9 zQdJ@yK{3{p(A2_+rmKb1%+uFkJVmG0NcPSZX^yF{t7cfIQ7A@a%6S*34^t2OCW9e6XT?NSp;GdeAQFq>CUF0@2U|+cr0>N2BS9fr2jSvVqtXy0ng@u$b-8Du- zcE5(-f<+S(aDw)E%ZBZ`(#<9@rG)|UWhaZq0gNzXjuCeVbGZuB z3ETaH21QIMNVh6<#JD#w*~u5{vlz&nKL7p=^h(&O#Wy)qq5Pir`gJ=AfW-j&nTnda zr#i)J0B=!2(HZIqaNQau{740>`s2C@|9JVl1D{e;HD1Pv^BA^ssI-3|;K z_~?j47#-1}@?)Y^b0Sr{d3zTdYFN^76S%Ab1l>dG2ll39(d9IK-I(Tha3PqIi2=a@ z&(Eg73fgh{_4D1bj1`D%@lA*kh~NH z_GtOCwduR?{&%=Y9l9~Si8S4*)BGKXhpNmiDYgE~x$&>{Qd(Lr`5hyct#^lqzF1UNt7WSV(+ zcr17cY-S6Q>hrsvzwoOS`=~O7tey?d80qznRlV?)b~wk}?#Ng6KNlmxM3Zf4!VwMS&VCoQ z34ZI$M2~MtH?63RkTM4+T!k=HL$C;M&Ow;z!d{L8d&ik{XnC{KcazY}f-O-_>Q5%r z3f(={frM&VUOg{%0lzos&B3IwPk{fg%7TaLwtZ>f zwU@Lwv*Hfpj};h~Mk->3jD;dDS&Ql?)K{~lYk0ly_yH;PrO72 zv5#fil0)0D=BO9RgHZ?jbNxcw7ME3=+yUljd5VE#3iQ_-Hp?|8jtfh%sG%rj)))|W zJ`>*iM8ut+*n4ufRhSapcW@&GK-656;&Xns|%9{)u7?d|^= z0cdye-($?b#~1L>-&=5`uO})CK*Imqs0OsZ{-K!65sY!-jRK;zwk8fnd|P)d!?-Cs zYVR|$l2!!{R~ec!+MT(%b;!uRi131;L51H@17AcfoWGd?_vIyTlzEALZd(P;x{Uzd z?aM(Wbq2G6vEf@1z3kq-G?v~pHmEeoljyoNGC#QV!v*tJL!I~drxqo`at*I|{|@pK zM2N)EC$fBKig=ik-`Z6YoTWrYCHui*UcG7*G}arCtc>)W!ou%Nl&24Kf)MfsU^&TR zWeh^W6(7b7!75xrw(f(nzLc&P6JTN)IXNX}^SQGfKK%YC2;+-PS|r_HM-8!o`O6Jc z^;JtM2(u7?peKyKTNncFck}8X9z1a_$1S6T)xRT!iW*~Zc}!YFT}6R;TM)}Eq-3mE zDijaJdV>IcXAgy3d2V=2*yB50tVfE5#^jHFc_qSw`T13^1C11XTz#0NgczxsNGZeD zf!B>bIb?^|T326vZ3Uls3G$Y?WX5U(6dA&EvWfZ>s>Y3&;zc#x9SGij2qX2q`ihsE52e72*zI#Pa=VyicqxHkN%uIS(xDQTGHR)6Hn5z$XHg9x zk5v2)S?DdpYj%P)Dp&+^n^1%g6wXM7dkcz@L|DJ6M{OP#xuF@)=yR$)o zFT@&gLdOj;y_`%#PIW_6<6`7M4Q5Cp8-oa$gzd4Rw4jn32o@C&+SwQ*jfh=Ge}rO- zOMKlFcv)k9fxv&_582-hJCMdSiHMgaqdj4TyW%QY#1hgjeEV;|LAQNr5wYXE&XH|2 zKj>+XW>wZs6|Y-1x%8!KsYRX6s| z?cfwj1!oaP05=kUhd2D0A;?4@*X{Ibxtm@p^W)9i$?PZi?aP%fD|*7--u%36p4(RD z5>{61>Bk+_TH_hiBOo9vRJEu8kSw*M?{l$l>sKI;mTEA>+r(@=OXxkrOl>`7PZ`Mu zz)quW{f0^B+T=)+-D&E)tLWK$uGkD%w%))HYFkz^3P+XCA{o+a=MEaN!?~&9lTc^J z!!TxjM{7VvXZRtHROzP>%g1TOO}}(}3fRjZA_S2Z1dGMfeFm5={(#^;OO|z zH@qSF^2UhkvrthvDK*`RuvoB2=R|ihZSZ3LkgNP@Pt9K!B#$$+o{^Kfvd5Vb*ml9w zPmS$$aFv^NM!t z#6DJ5Q-Vi*P#OBVZ6s)+h*x7AZgc=Hpa+I#;9y0sYA*lZmN21!2Vqu6XkOD|z+t)h z*VW+UeF?YHbvquy6WiP#{nD~BZO|sDJDAF;qQQwAv`Hv_<@c_e)DOu;>pqJP@(}gD zy9(wpII0;QCbz2U>y@kTgR2<JE zKkNLG@({{GG_rAXD+HVBhBJYU_>-r8dYXK60CjYLd;w#`AdekM-;IKe5@&9kFgad) zO(7eTtb4Jw{c&&c_l$;iLsBH^7)lXtz++Ea%4Lr^3|pbXM4$@I^e>jJJ#Vzf*&dgl zYU+NwsRZQ(BOe5PoE$JOVxBMD^bn(k6XyK1iqSn9QEOr)N;EVC3ByeC7T@UCoA%#y zi+uMhRDC-elInok^Vj2br*r-rZ%=%KCDS}5>A7vY$2F3_+ax}U-rC5#OZ?Kf)xnkdFdc(w|0@E7EsYE`AUGI2%3nEN^Ht8i8`ZzPAQ)iN` zBE5}4eYp_&%FWe$uclm=&$^6U36u3Ch9YIM+jMplvv^Ab$EED8Z8378EY6LUCQVaX zhgh&7qqYQ#rby?wSLL5wadeeD^2*Sgu>FC%!c7~)hV)?y9I3q7{7=Di?bxVPL6<-e z($UlFs;#ew0(DH)T2=r?9}W}74)){|Y$^E_mX?Ta`GL5T&53r4;B$ds>&D_@Q|K3XM1e@M^(uS1G-YO-QEh}-}B-Eq@o<#lvJ2lngIunP)Q#2$nJtsgXIVe zJBC*o;(im~MH%A7NHv=aDDQL66ky<0#Ie{lf3}uaYk$iau?*Vf;h(zMdwO!WwYDC2 zN|5ov_tOLF3Pztq_4PsHdgSAu3tz$O-Z#uV^;#7O1h4-h_9DOh84>>WsG_GqO3a1B z7`lkTwLRcnF>UL}{#k_eWow-tPghFd-Q?vyGF6@`tiK|m3k62X9AeZy`(mF3d(V&E zEsTE5i6r6?Zxs~y9d~oAw^0q)n5eCC27nb#0W0(#NM6sNbEX+63eQseHpT5+Kl}QG z8;+;`%PGjEI)F5c7D!C5w|6OP_TFg;h-+`4kF$Eu~#CtlW#B{Z{< z1dHTUxTVSZv-+Y4y7?lh5A7?`_%{%W!o9(9;=fRGg7NV}wt)=q3^2uNTLhvX!IXvbQ}Gw*N=^1{ zEz^TX$Lzual_mp@!HxeY%77(JNcbKNR_l$ezIqb!x{dSnkw6_eq=ax1W73c|+qYJdO-lYNf>LYWyBAj;>w9w_4G{3UzNdZC|HWjLu`# z$&jH=g~X*~|A|M$>TWsWd>K2k(zyQA$TbV1O}j2T^SG`B6~{AC6K<8?lJrkQ(7Zri z3K5 z(>@}SG`mSny)z<9NAQ+cc8!P+m^1S6aya76znXF}cO;Cw|HA3^$Bw={X84Ro21R7% z=lv^r5RlccTh_19u${h*N}+=&yfua4T)oJvsZpv~wk9MCTF)$wPvaX*G1Pa}=#R(D zoiPzlm}*GUEK!V5&@X%EAw9N#@&;&TX|O7=waAP)6Nzfx9b{QfC7 zy;dYn_JGXB(HW>Eq)63T6|4LdI8&y7lFYQAd;}sjoX`Bk7{Cz782cQHjS%OCe-n2a z?Z|mc7+7h_eVROlVwN;Uj-yVIiVAhrdHH4EPi>n2q22SIcRyJ~J1`j&u6X@h3PzJr z%-D-ixTUKIY%Faene97d5hPivIhVMlb}WPsw4wZ(B9tm3 z%aHCm49?Ut#Rz51XPrEMm64q0e{UyJHsM2y$~)SOy7F%W8=mmC`ts4F3@ka!42p01HnxN z+@dpb1C1&J$wrJp*~&04ce4seSYrNwi9}&veM716(ev!V)$%wxBOZ;Lr4=J);dp9U zPM_ECL)Blg9Dn6Q_fwA0VG32HYrt-ViCqbk0iC79+#WgiTOe%~y{n)RQ(C#Ua2*e3 zhv#IXcjEOKtDuo#!}(lrQW;W;!QE_XHuRF3j@_6wA`5Z2F3G9T$Sh4TF7a)pB@n$_Gb!HYI( zwqp2i3H{9c6^nu#MIDE&BjpYo`0g9$^ESx5wl5BlmT9g<8}Bll{gI+{8QtuIvrJ8;PRR5Bf=xa(Lo*8%DN#>Z9rTL3&3kR6PWOqtwc-f zipXcUYcGWcBmXuxJS9f#a}Kh*N{=(CZ*PD3z5j)Zi76D2Z%(HaPr1Cf&`^2idG8^( zzH8?->7(bntl&ANxq8ypCXZz?R%A@Ch?S5k9E%v1iDv0J;-^C52@+LiEAWb$zgW+?D6q3`YQmIHN@od;nEZsE;NIK?; zG({dtSkT&|_jxIHI~9D5Sco}YOueuPQeEo*vH-7;$(+1{ESHYeKSMq*K;#D;bl#_O zj5KuEQvYsjRB>H3t@|CkYCv(w{R#TymF=|hYa(pmD)LEc&oF6N$-WjL!P448y zY{gy_RKq;LoI#vT{jcN&r@#GM3y-vQDj6Q*j#Ev@b2iHR>XGuc_en7yaZKd&6DaXJ zT=3h`MRchJQiz)7qWh>Qq!d2`mrM4236+F3s*xIYR&mOG6SY3Lh<;B%X})tkawojgM>Az#P*mmo!_+Q6B8+tT z0qfp*p{AlNZgQU&|KueIju_nGHC{^^)Vau`l-T~=7wjL^=0jg@8yD<-`0h(WyV6hE zA1%dfV6NOUL0eNR9?#&_#kyN+mW#trjZCX_m(af(d({99QGA1#sP!}F(=#hm&#!NY7}0(rw7c=iV~Ib3S1#mDDyVy=o_U`Ut}Rc5ri8ek z-693bq&bUXWeIa{^3{e9Vc6!CF@9U!Yd3Q~TF)+~VY4lMwWne!gC&(;u*|T>jD-R+ z8lZnjhk@;HWxd7ei@`3%fTu?a1A0b{CJ2%|g!iST_WadBV?|r`A1!$aTIYS~^nM*> zG4geGW!PkBg>!^ly6dRn9(e{cNIkYQHQv_G{>1y#A zBqiFKT8HFiT+ArF$h#Ch8RkWvAT9)^G2Ic^pw-pYip#o8-?aLn)@%E7eX@|b!wOnh z$r_Du)wpEnrH4-Q*{H`h(ka*b;y&RCH)I#_?Xgddb8C3HC9FXwOpxu-3nObXToJ{i z)k6XbW22lBpY4%au`km@1+T%%=zBycDuW@fw6;}qbM}e*W^c$viR3!1RZ#><+9b)G zZ{kHc)0|I^OmDWSl>(?YTp0mlXZf6j-|HIQ7Wdgoiwekn5J5V$xmLoxOh=;6l4s}U z{v@EsIYh*j|Cu4)?a?kW-9N*L_sj1AILzA?BoD%J+>wqss$G|5#cvy(P+^+Gy{$@O z>j;OTb+vbJ4du!-$x>!FumxrB;da@OiweoLIt9Mm+_3Q*RE2TJ<5{G>1eyLhgj&iZ z2hBnaoJ3Aw_%vK9IvAc4Kd z>%}1$)TYySw{Wq*76r~o5vb)l%C=2N$OMv5Eeu6<8zk6&7>^4eA1nHYBa*W>y#Iz_ zXij!gqJ@`ZA}ym%R(uX?6rU0-FeZexK`io*VVeW$7&bfH(bfe8g>T!|L!kbfC;Lkj z+AS>n1DijK+8oIlu1qTOPhQs0MzjFQW z@fyZ!*%1&TCLo|7jsY3vGN?fvty5Fb*3R>E4^6~v4QITvCS|rPJ8ND4-S+#*^~~)x zjYP4$Z|h4|2d*@y78tc9lww&^;x*+piNkCisXN6as4Td0*EwB(k@&MFmcG3`1nN+y zQ*{c)r(~KRR!s`3aQ8Nr#-4g*v;$fi^4DXoOxM>(N}cs(QAV$y53Q}BhY$KH*5)D< z$-|fTBR<*Qbveeu;&GH6+G>t35tK=?olcuaAu#Ff=qu?WRPLY?1_CJ|Pfy3GgM2Kb zh@2Tue6eCZk8qsRclR^qiPl_^L%Xc!vk*D76En2{2izk!%3C>T9ct8s-SaV?neC*5 z5tF#^d^JHYEleo+O%nf^*G*)b`B#hXqv|}10I$6M9f$b8J+vDwEQ43Kqmv0TGr|nC3#3#s$ZO0T>0vh`PFoZi?DKrT>17cW)b)&UXD{_NZg8nw zDaOFvCHZn`O5_@w9kuRvH6jD$eNz)&Z~XSdYr zB@YjM2iL)=hDexR1bk5QW4qx95tT3CP*qe(?Ce)!@i&PtQbtV%i&GxA)=0l*5}3HL zR?yQ>dK=c?_l2yv+}m21Xn!RjZdzXFhoH~Q<52y2xZL@@_@Ta7JLj#Ev9VM>)34pF zWSk8W?ZGYy=(g*Mxedy;T{%`v!4=0JQTFjd6prVt#L>Nv$0+GVtRD#Woc{697DVsV z(m!@E5po6}iqwd)Wp!Ohip1v@j3^j&beV33OYjk!8Rp|Vmv&=To5%I{$w2}oltx~$ zeEetyzCx`S%~l@&FH>XM8~s&35p|3R{FxZcfC2M(+>Q3`H)eyw$sESvF-~7bu|^pN z*nhn#{3t1s422WmCStrplC!fhPp{x-Y{vfMPMi{W-sq8O^%H_41l=M(bj>*3hKz8$ zyFr#N-5S)1NQu!CSK(}3$*}uyiX~%0WtQT0^9xO0C#Uc`GPL`ke)pzX_AbM^bJ&Z| zj7H_6fD0;PeQw@=;_{iI&7oy{XNMewl#_}JqcRjj(4s@Lpr%pntf-RU8bKk*9CCBT zA~TFrT;g!3Ohq;P`pGmYY8(>`Yhz%=kz-Ux-r2qWt*W%wU#RWWz-8z3Z~tQ$dE6)~ z3)hen!&qUSX6Ka{a?_Y*#T3WF)3WM?sn>_!B?yuafrvTp6egZUv$} z#Z&HB+1S!DVs|J;omflc!a5Nw#LaPRBE4CD^$*yGfg5#l9`Gp5s|DxcgqH zArwNXR$%_EO8vYtf_%I66JscY@IUK_17GJjF}kts4)33#?MQOWh}rEd{bkgzQ{}KP z8KL3SbwO~tP$`jQV>ln~vpZsm{P&5RJUA{HhIAa3AyoOTDz9h~IVYosnz^#NO%iB- z1eDj<$dyLLx1+anW=yK8Q6!JYwoElvQo(1Wd)12{m)?+JQM3;X?=5+^yqsgWz6=6Tq;`yj7l=97h;z1;mgvRvxn5xzyd3QtJ zsmHps+pE=&uT#gn+(B4csOSePKkja~yuYDNu5EGPb1v=!{ygRE6C~KMWZy8wp7)f; z=HcOD6&mihXd+#us%Au+ocCkHvy#luuWxW68;SSAP;+%hL4Ra!by6FJ#TQsQb-7T_5BQvr+d} z&zE4H;%zI-7S=3?&BE^1I*na7_8GxCN3pSdsG&y!zj3Od`nzNG4Tp4k0!|7k2%Itg3w> z9c)9G$R5PzBr!LkCLj^PQ^-;b zd6A_sc*Mk}78WJaka+0!L#w&Pt&wMU&gXX?f}Qs$Nt7kLV^?X!cw~~M*T#t6Gcwql-I%=0o^!m39{KZPkRkp*qqe$t%xmDo6C)^to$o z3&pq1HbdP43w#jO*J*Jmyq<>G!Afe~UOthAOaIkyMsIbyA_voN8uBRQL$ex-NGH{r zjW(H!)6TaBf9+e=Qy^f<;G(8jC`DAzR4 zQEcKcbwg{>>mB#Gb3;4IROIE1WsGYP=DPNrc=F2xJPXLwfvANYAMyo z6fh3q``i(BydoAknsBcGb-qM((?=bb1xux+e#7*Zfo3U> zb)1rd;ty`V3jsS!{q`hX8^4^ZTj;GF2EqTr!qUC9rBBKKg+nx{Upf zl(x|mY7g!Kw)$-?b>ZFowp+6tN)@bXTh+8Y=QT-Xl8`S^0~p*3S|Zkg`tYV+N#D`y&B~IU$)$yZ{?x8nfRu9a;`H zXOu{dF*%uLT+M1CXC(C6Bv)BVhL$GTyXQ1LqZ2b!$%mPb zDS?$Yl_QxnS3R|n5I!qM8PTm}Q+(kcL)@ska;FZDxOS+y z?gkz2VPiJV0hu&AZ@&!*=HpkRAFL&kdit$jX?4~=PItZA|i z+r0=NV@*$cLSq+#CV=zToeJ5X$>)73pWXiWdk_OHg_21@oQ0XSS46N#Fr}7k*qA~m zjGgi==?1Qkk$>M#`KFX^YK6FqC;4BAo8o-7hR4nj(Nhmv|ALl-H=h_xn!>DH`9VH{ zVGh$SgOB?hu}N-!>)wStifoScJqgLdY@rPH$xFL1rYH`jmyZ-2g=~Aswkmey03HDW zJLBoq?{z0J5v%=ir;mgY@{opXq`g|{&L5Sd&h++Xk!mma`!!wiqpW-x`o><_$`e6m z&1AzAA6Jlh!K=K)=irZY;;f_kwd3we78LN;PJTE#+Nn2Lz$ysj9Dph_Dk>`JZD0*j zdvsyc<643kq{A%@ex1Lqt83Hg!Tuvry|_sI?<|b(W7|$NOK1BLKAjp=abL0=Rg&+P zW3}LBgCttYE1nK=XQIS+DI??%{Y^ z%d!AQpJPqU%Gv`ntk#j>k{ z0?P62ETU!s33le7a}hP|IYr~N*r`8p^4f6bZ(7}q&2V*N>Zly(9dGvKK#op3{D*RP zqz2n5nlL1{sY>D&Q_aZMHX3ALlfo4>i^zTzRP;3M(}t1@XHA8#*6jR*atP-slBelq zr3Z~sZQCzTaAK+zXfRZzbL~>7kRVr1P_=Mzh|;yyk__zCJ38qxcNlDbiWud9AJGYy zoh&DIZXt9Nkl6SWJn^=5wYTWj*1{qjWIMs(OrWIbAjli0jalQxYlv~@jUbbOP2SMS z>8DQ0Wy_H@`dhp5ilh(*DR@4nx%-mZ(m2V3&lBGljheO{KgtF7c-4sU>S^6L6&QB8 z?hj(IJC4o2bdG`HBI5A_gq_*zCRl{kVMv41;ZOYy74G3N&iK0i9T?FEYss)|~Y zc9X6B^Q{F424tYu&m20@4in*l}zUNx4BLnM`8s{p;@Uv~+fDgoGP&eLa({Z7$jH ziZo@eVnSe7ie2>gN-?%k{}guSZL?&>@R%?H1#v%}POp~hY0^t#&WtSvufJ0H#$H#B zxy*0>W03gZT{m0RtLC<-5aiZHReHq|sh<*%(jrgt^1(Yg*zMemR=*T!*MjSd!dh|S zAVMKH>C(Z~1BujZZHalhnIKsi6@CT&*~kluep^a43O$-MS*KGMgbN!$_xQNdDB9Y> zI4T*aKC2}yFKdEccVC5elMu;1uR$hrx;O$sC{&192mMvBELGp-C5xk@V^Jc74#{`^ z)1l6g&Xii{cJ->Pj5&uIJJXWd5C36EoCgVPP8)u7${5|)73z$vyo$jS%_Z%;5pgaU2p$>{#hR1Ju_U?6#dPpP%-`&-Jo2PBBFN_!*18KkTf-hR*G${| zO$p0ZIw<37oZC%x(E(@u z=#*5*G-tYatn_iXWm&lhVvFRKrjLxuB~z}IDGFf=WvxyopYNBRBw+?PaZTNd1UNnQ z;9OJc+X)OD$rM1@>9f$g76;BNh)yqTw`>R1l<#~kV@^hMB1^aIT=_kloVT2-VvTAv zet~x0%MNbWkB+$ZZw*;ieev*z17lJ`V6=(8v2l<&hb0}ROH*ig(3(||;sS!Ocm!o# z{nEkT({O2|o&~V?r5~4?GNxnp?^sBfy?Z#=t=5v&UEr~I&#Vc5vdjXL4tQ7bKh(BG zRxcWm^J~r)M6Hx4kimBncvWQy*>F*#$ElSua}}~MT>`b9h7t%!i_mO6jfYFeL!-~n zdk^f@kbkHur|E{-n^0Ya-=JFs`#)j*v$szb^Dz0gcIITKuZ|J=o1BU<@=LP)hcha6 z9W@o&)IbL=bfBD-R!@l%lbGcM!Vt9N(O8F_~E>mKSi)$UebGfeDWMU zWE&zEg5Qk|yLI$d*CDECh9!hg`185zMbi*;$RjF0H9*y$9R1EZ7^zktow$bX;y%P9 zU|VE)Nm0sOE)3WsN|?&RZ{U>dUIlj4zy31M(Pxoz)qGkMr5|jVCMK~-x3SfDGknk0 zszp$?LrVq^I>i4Ce)$#^r3TlE;ji);%ODPZaD?S`7$|;&!EG8E8Z+aPwPIh*f)+eE z1fCk0b;(k-Cu2u{$#RB+&7X1N?-T4|9~xo|x#brjCmbJr=-*x&6#o098jW}lzqN7J`XrwI=#qtnYD zPDBNW1(!a#cDxpkH%Do)KzjN-%PkixYZ_ks20n`m{ldJsir#RgnDyhwG0^>(n#kaO zE0Q=yfQc10%$#HN?qt#GfnFa-d5s*k1)R;tDshnNOnO$HPD#EvuH`>p*ME*yMnCJF z+;PGZ3FCTlH7S$mq^Z*+sXo6h#!J=gLYy{{voqlv-6PngYH=C9*o*t}<(BK|=nN@C zDp%Fenm0l&+5Py^v~{(wXfYGuhIQ9X6Fa+Vh`5gE!?$7Wln!z57FX8Ljq}o0b1;%q zm7nU?ZgVJmqli<9R2rf~tZit>rNyZiM;TreLaIP~Jhyn^J-XO9Qe9V&y-_KtCPWD@ zwrer@CBygw*zAV%7IaF7g9%i|nHZ{M&*<1zxf<-P6>wK=>vFWCi{Gc|?q4P&g}QuY z8(Du73z?v!Mf(l9dX?_L z52>uzy4}j^T2Oo3ABV?d88Cnz0;q=>Gb9)3>A~A;ZGm1Mek-30-Dqb4Xx(dmNi80l zejMm^S3^X2RPs;t0?3Jd`{$(!fxYjv>0um* za?!e1s>x-d?1W3Rxz+w@WZ|tBMe&uh8BzpVLyAY8})6mM3fBj5u zeseBJ--UJZh=3hqoam)fg-U9t!>Ue)B%MwCT?3tr98!qw1Vru^kBdHgUhWju2T&os z0j(T|KhMt7#Q~c_7v>s8Y&cS0(AjQ_+hZ=~t`|1YsiZPUT0PQ#NOr;Jb6Fs#Q-_AT zR?6RO4;ai1Y~3~U)HlWayDcQLM)$mMM=IN}0dpoIOP=+Z%5;b_4IMuGzRdgv883X( z?qlh9^z_4Gvak`y-b6owKSTTQ#IjO;9)%3Z_xs?45DNOVi*)XG$JEH0FvLH?Cr%gt zDRnslD>XGvq#8BmnM?5by~Mo>cV*gx5ue^pA%^X=Acy9>91QT8UIu5fUQ(*gO}=U) zwJ~@bKF=&~nRtH9ut4Qc9=H4Z3|+qNGfC13PuT3}guny>O1=Zls&C?$R=(&dN)DrP=)W<6O0^nI z_T>$GS-5`GqfZ^5bj{JRE2tovyVc zfFcHHKt#c%1r%AV;pZ5UXKN=&Ny&Aqas;Z5&k^3gLwzi@5eQwEkl_(?wR&}SXS?=e zoZ5i_R~1WYYK}5@(q^|Qd#PvtJo$KBgi3wuOl9%O&MrIu`BF|qMu<d{epFR)pHD-$#Hny(>@2J;CO8l%FmAW0HM{~u?^%n2Y z`}Q|39P0|2k|t89@N{IWmX+6Kl}yNz^U5tivxo{wiv(ZIbU z&+0p?Cm1tB#ib%k&C)ZajT^d994^C0P!r_~vW{c6YIk;zx66vz{PS$^BJe;o6+U3t ze!pVX{^x$R0VCA^Uvy;xEde+3NHgt9)Q?%lv09a6lZ>kZBr;>UIhvHQh7F3hxqXtt ze=%;!SR zbbTJi+C~m!fJ=y~@To-r+lcIszD}*6&FEXF<)sDu{-f%+N$hjda3JOHjir3>dEo&g zAvBZe2h6FbEl25%--G(l;t7})u;bA3y|IxiJtM=y+8T<7nqNu>Zfd}V1Ho{BIO#$< zTB%o;&Q1m!UE0vACnkOL0`<0sBxy=lAbo+gv%ay>kRAR*6dLTkwZ-1)G~e7vZ)qo) zX8h7T{?y%^XfoZ#C=_$LAaRo4xz(}0MeZ< z?f+JZ$uk&KgLo3WL~!qx=eb55?%>bq*6ymxI0S`G_*x5x?(I!s@4g)njD|#YbE}7M zsK3AYMvMgTKuGPY=+k=ZE7$F5lG%z*8q_rxZSCHPGUQYg#l3dt-Chns@aOFse4=SC zp2s`ir6&@pucB!h_EjROM287g+K;OOk+@JN0+5QQh4R#}bS1+!Y?0j$@+x$Xk6_{e z7=ILGgn-58YC4#X#N1sm6CS+Yr+5Ql5fm3}`HrzJpwUOl!MSc65SWoLWW3zA;3gj4 zh9Ogv?<#6B)VtPiurPi6U~RrDSA26=x$vI48L&-o@XnL9iL=`<_k-gM^6>G+_buF_ zjs#Z*m`}uF=`=&O5bl>0}ZjmbvazFCf41_F2ItP+l>l6hi!a*U!>2sr3LQXkpa-*d~X^K zpNV_L0f#IgBGL)h5NJeRibO%zG-LG;9j?!&o~^H_IcwdEPGm>>@oCh3E$ULB9tzoY z%)F`K)b9LZ?h!Llrq@Bf{8plBF#ca`zpAD=z7#po@xCmH4aoeeQB*~st&p> zS){vLfVCzYOP%W_Kk}RlTB&z&ik*7mJnQT!sD24WsnG707Qd%NfiC=r>7okb6!^y1 zrjozyF{Tm!m}*D1%>A!tXgueGM^P{u%GYm4QPeU=*o;>@JkS%Tv>+TJpwgOR|%$tqN=~sI39qA^Zd7t z#zzhgeExgS{oBoABwzPcId$~TMLu0*o;e)mb8_>cRltlmJEyMCkw|QYFjkZ05MPpD z82R1it;azdT^#fKiV9{ioZv;|4?WUqbn`cKlySXloF1Rr)t;oXt~=B{j&@vM5)ryx!iOs3AF*dpQZ)v0erHSwNhBxQ8w3rwdv$S72P^=sm`?ImBW$aHohGOhz zmd&H)o-Pd|^=l|oN>=R`R$|t3ZVx`=lFA>lg5mD(0(}y#fH9TYa|Li)Tju#;=Jss- z163;Y#hZyb_EBn^LsfjveiDS}CUN7p5i`e=glS@_BV*%Q4J^X0f&-5iOk70kcs0D= zvr!aor~X1(qs?C0zK$2R>`K&0rwY7_%IbGbncu=c1w=UgqJdCDCksZ-b z?3e-JZXBby+s631U()+bzf@3dQwiJZnwfgVWj48nYv0g}ZMYCnoPv{>cQM&YCmL_;9HxXDOA`#!#5az|@^7Mt1VP>cU^kk>r3e1q-@d8M zQE>!h;u%Dugs3AF-yEhj#{p>d5XxO9=Vzx3Qa;y1TLV!l!7KO@t}gV#963)@HOAZp zS??o{#n30;_4dlloa>f~foNR~eXl;LV;+b4O~vSe2sB(!utGpk92_0(3A{)&(%>!p zq|!DHo&7L@MNqx;s`^H^camp?|9mlisZO?S;o{k;y`7Es1Z18aEfIIgD%M z1-;j;LQ%5Dxk(eA*5t>P?rCEQP}vc7_&baHNILY!I5ce7aCtV{KcB*ji+P3Y z{>Y<{tdt~EZguKw8mt7H=C7+XjArr#%>I*NW7`m;gag{ZlJ}XU)n^NX787kXO=@Iy zI*HVKH7pC+=K_7tQB0$tKL4A~sKC+uhwGoU*IftyE}GqU4_-@q3|bYI#rs^-=3=Cg z4-;n$<5>6Cr|Fq*=|JN3&+4ABO>>)rbXmrYW1*R1O)~4$cF1L=Po~IgT$YeHe9K6U zGbbEmJ3C+yFHcN>4=@`_g_;i@szZV~n}QwE_1~Kqn>OQaOkGi!@0{njvbx%+4yED| zLp}-6B&A<@;)$g@|<)w!h(dIinTevvzn1aBlL1o>+6Xb6y?s~ z;PtG$!Z$DF@G?h3dh^FL(*@sTCjw{yVzbSX^P`e(Qoec?_+7(;i+D$ZOCySn%0gl) z#4`j3w53F0_4|LlakoDVgbO+R1s@V+jQqR1ox)AkY5M#fD|+v5eVD!xj=uIPcJzBy z%-&`m-zv*P0q2Jl_fKazrCI5sx8B2a-|Fv*P6b#zOSZYr zY0sa@`}baL9LnQe-CGBLD?Sh+y4QD@8Yn#BZ1XZ0q`5haOzF)pg`L=IcpEin;B<)J zW}0GZaOpw>o9-?WpxrS&h1a6D!aE-JnDgfWW^@p0PJGh)W6s6e?={B#h#hDNHks&s z*e4Y^5&hi54p}io{^YTlp7XT0APMFe01sG;)l6Jk3fp+{U@cGQk>`Fp1H1Z|qazkQ zS;>9!J=fRgbaOm-$b?F;5{%n|7XhhU`tR z89{y>QnC6+#81~Zn&8* zoR_#OU43QFUkF1U2{5F8AL6GZ|E>5;k=%jBwU7MvSHC{^{S0Xfxq(<}GQJ-H1A!2b z#NPU!c|Lyf#1nAyAFM)$Ow1PMulO0qIg0acx}B4yq!#@`%Y0p!b$??w1$RXLJuUxG z>gk64zcau@nuPh}7*UUnXS<)*siVz70#il_Wk^eEQ?+mH*WX6)i=>(J8h)Igs`9^% z@G!MWUiV+atE+NL(aK0zSVEN<>9<(uZB#RKW^N|Bvl4&?70RFXuk1T|Jm7_xrzc%%=$kdIW_(ZkXAWz8*3 zV;)3jP7=L8PWa8VBh~ual4C5AW1br*&#TP#%=R3OpWAcPl%oa~%eE6vGkj(u>wf>B zy=(Ai+;OwO)bQyAVqyvAZHK|pkkB;=)729B_y zO6iss{pJ^;--=DBh}#S9$N^M2&j7=ffva0HF3HbH04+$vXEdAg{OuI(@q}}}y`xfi zA;KcTrd9i7ImqT|-RG~SSee=atzVzSC0I~Hne3FbsXiKeyMh}dXX*?i1L|gmlV;A& ztUD=TuChot3B3%nsUhS`m-3}!C5Tfdi0KFdVupQ{o(c>lfQ=6X<{-xcY&Y~UEuhN) zusd+!0b-&#Y;;g*-jgowy*j`?ypDfPoNgYp%nDDh>eCFzMxP9MsQ_JccS++Bms=g9 z3TC9v-C=h+ksVPo8vg!s*<;H89O!Q%(K?7}=JQQvPq-B?__peB#NSQn7O^Est4nqK zW4D@~{M^MUJbUT-b{WqhCIXA#+V@2u3=&_8Q`WZlx$Ov+6}?YU&rlCux#&(NrTed# z3M1e%BT*^sox@*IQmruZUWfvMsP&- zubaSYD7616wvPssxF~JWoN%&3p9?=r|0YqtT`nIEB_?d;o+B* z>|I^u0hZV4SNx@*05Sqe4Id@}6)>rTsGe)(2y3>PV8$2BTlYwJ*T^1^Ds$+Qrj1C9 zJ&QA;Z=m*oDjz%;Jyi6I9IAthS$>X=aSNILZ{dAxJU?A`-dO}QD`MWH56kx-dJj^Q zT$*245OrOq?33JQ0M{<2wrN!wJ7^F;&-6`%;hHN9A_0910;4}fSF0J^Aa!9g4GXiU z5mZK&e^oNt!D2BQ?}1!CxgdQm3VG`su=Lyskt>tsvFsqqdo#3jr{NqdD9V3Z&l*#M zp~zf$lvIO{_kiY)*uV(h;O<;*4zop4ulD}SP7i4Pzdw!(Qb1->S2UkT|4TTK%;HVz z+3e;ndAT&m-nieezIiYc1C9dPy68vNDlJMi-6ioAxTM?4hjyI8g_A@kYRt7lA1aV3D3#ubLkn}1H_@E8AI;dWkiLK znj0-vUMhxcroo1aubgbs^%@$du}BmT~Plt#))` zmYrC6*q{ukvr7=0dce6Y3alOo2jDA1Ixan6x9kQoG&pPkvN{EPxpXnt2;;M@B=9sC z1Xr@Vng>ClhH)F-8TlGfSnNBkLe+@;Y=>vZ%!^zGO|rE@L8xF^WM1kbtIdf)g@#2W z>e02wvl7fl_NOTeOq6PHX&wrYy*c zK_@n?UqpLL?nfBq$MLt+Dj(@zDE>Q=j_#cHzsKvu$K0T#E3iz*8F)@U?jan~c*pXO zvT-B+r`$BGDi*s1q&vs0SpdBf!~X`(2ynPQ{EY&L^Ph}@MLhl#GT<{qh4n6UB7rCh zU1DO^?vJY}1((FPeBH=RTb3_!ZhRy5%&2d!LF{!Pr@%f{a^a#4PvI{#c&(C<1j1{V#ks_h? z1C75Aeh2^c8V_!~*PUyX%2*0T3r-~BR-E!!QEExfdRRn+S{5^g# zVgK4{xO{{rus>8iH?Nx#TAD zpdby1XZApuB(bM1`*1LaLxGPg0c-mmxoFpCvBLdXxuxZK&x6lIvZ5w!6^BF11J$k57B&Fh;&m2L}04tS?*v&x%CV5vr8-*uL2t znklm%|GU{wTA%1N$c7;OAcM@o%m4>cJnE6Tf#>AC@PN@b!)yGK)G;00X%#XhJ!+;_ zC>~5{sn?QX2+E$h@DXJS(fCAIutfEkn|3XrjBQlVat_y?#04uL!P*m3@B{S^g_sMZ z;Z0kX$lrTj^OPe)L(fu3hOb$}%u`YyWiln8^piVVE51No7==i`a(t09{B;(Mckfpl zfIG)2>nMJeUAN^i_}>RgdB5XEWn-;(pNF7OW4xmmbq!UK!7Dx^tK29E22}F2s8&P+ zd2`Ce=;0ygt7ERBbu!+4pt*@Os`{PbRWwz33r42m*NlYHiWn(MVuSCULYFy2SB6B6 zzu1Zn9GlFuLkXCm%6gV=6CzfX(QlX50DZ-Q7oQSQmbWpbu=dLe$UG7i5I8<=QU-H3 zp-X97&y>-4teF=}I&`BggRcVbZQ3y&f1C{gi~kXcQADu{s^3Ua19w-oxZrX74h#P~ zGva6&)MERSnOMWS>qV^LY>9k%xCOgRm8leb6+m&?wXE3=H@0h-$W)Y(f`s)RlEur# zElxFRU~j=5PFWil^Yb_E+i+zCOHt-saQ}hY`S&^~VDwDpZ@N^)RQudht5)jBHY{Ui zporJ;(4aD`v$H%W}DJdoFEi`6O8D?f`;nHx;sC#ppV6Q~v9z-bPRhn8#J zP2o~iCja-wL@4LbR}u#Rpjc5!$)=+e?k9>LG zO4KJv7Rjon>QlXXu_=1k3v*zd|BSRL<22rjI!+TL@fjEU9wBm>O!Dad_c{L!3{yP1 zqymj>oZS$t6W5~}V!H^mgI@52j}Lv3A1C%|CavfT8TX8|34d!MV^3R}sn;^P2{1NW zmO@Dr4dTYUCMgrHt$WSQc^>_RTh~e{!#G=Izy z!j%M3usxA4?!mOVQ1`|ztEifW@Vo&^^$<7%fl)wV}` zh!e5{)+07^Qrwfc@uFQ}m`vi8W+>d`0ESd>2OB0GFdrDQtnkExrcgZ-C)6ZXe~*Fi z!)tk*BqD#ROy$5(2+d1xQWAf=NESDNwJueefa~WNOO+=C)p;$hMLne1nsOOY8;~p1 zF@?M;^K|t&uCfGep+?j1PvrR?on>kt|X3NIX0UJpUeHkH$kMTC73H zm%r3P(*$5j#ul*Oa$)>h*PK*T@Jm&g zN5X?LkhU3Nd$C@Tjs;w+M~H}s#<5sy-lY3(HBT|EQ!;0i_eH@R#C7%<jxz=)q8w$*;jFKZyU5ESl1?+ zGQo-joC{!lrL)}j&+x5a8REcvD~P}#%L6GVq&jde)Oc=l0E=@9DjapEk_F9p8`_f- z0kV-SxI6c(A8|sl(Ui5qsm3Y?k!_vZ+BBqhxKbmy$G~R;_xR@`%%5C~=}_T_2bw7^ zD+8H9N^&yLJ0lJc4@b{?kw=S1;)Js4{5go8hwX5*tWZWb3rB}d$7)tobJy0@Ev>EP zm6cH_XkJCfcA3NeGqJUe0zW~=a}DZoiFvt6-ATb(^+u}yUEZt#qXiFq7!RmRnO~mL zEhyx%6z24N;L9er&LFhU@|M@v(1u;=(V3M+bJx3n^xcDt-UA_eS7z4p>e9Ir2FQAq z*~=Bh90!!y?eXFlx0dUvL~${}RH$w2EAmDDFw^(4Os)Y$hQAC4gx(FP+i_e*U%xXL zp>z4xi$}e Date: Tue, 3 Jan 2023 05:13:59 +1300 Subject: [PATCH 286/551] Fix typo in README (#272) Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a214377f7..2ba7ba828 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ We use the `PARTITION BY HASH` hashing function, the same as used by Postgres fo To route queries to a particular shard, we use this custom SQL syntax: ```sql --- To talk to a shard explicitely +-- To talk to a shard explicitly SET SHARD TO '1'; -- To let the pooler choose based on a value From b983107fa84862ffc9eee64503bc5e9daf88373e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 08:15:27 -0800 Subject: [PATCH 287/551] chore(deps): bump arc-swap from 1.5.1 to 1.6.0 (#273) Bumps [arc-swap](https://github.com/vorner/arc-swap) from 1.5.1 to 1.6.0. - [Release notes](https://github.com/vorner/arc-swap/releases) - [Changelog](https://github.com/vorner/arc-swap/blob/master/CHANGELOG.md) - [Commits](https://github.com/vorner/arc-swap/commits) --- updated-dependencies: - dependency-name: arc-swap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9663f021..6f715ee13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,9 +22,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-trait" From 37c16d531ad8921ce729693dae699b45fc5f8ed2 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 2 Jan 2023 23:47:31 -0800 Subject: [PATCH 288/551] Fix tests (use sudo) (#276) use sudo --- .circleci/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 6ffef8bae..c27bb3b99 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -89,7 +89,7 @@ kill -SIGHUP $(pgrep pgcat) # Reload config again # cd tests/ruby sudo gem install bundler -bundle install +sudo bundle install bundle exec ruby tests.rb || exit 1 bundle exec rspec *_spec.rb || exit 1 cd ../.. From e2c3b4215066de5e44933deeaccf7c4c92e2dd8a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 23:57:29 -0800 Subject: [PATCH 289/551] chore(deps): bump sqlparser from 0.28.0 to 0.30.0 (#275) Bumps [sqlparser](https://github.com/sqlparser-rs/sqlparser-rs) from 0.28.0 to 0.30.0. - [Release notes](https://github.com/sqlparser-rs/sqlparser-rs/releases) - [Changelog](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/sqlparser-rs/sqlparser-rs/compare/v0.28.0...v0.30.0) --- updated-dependencies: - dependency-name: sqlparser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f715ee13..0d160cfb0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -980,9 +980,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.28.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249ae674b9f636b8ff64d8bfe218774cf05a26de40fd9f358669dccc4c0a9d7d" +checksum = "db67dc6ef36edb658196c3fef0464a80b53dbbc194a904e81f9bd4190f9ecc5b" dependencies = [ "log", ] diff --git a/Cargo.toml b/Cargo.toml index d29f4c753..aa853ad92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = "0.28.0" +sqlparser = "0.30.0" log = "0.4" arc-swap = "1" env_logger = "0.10" From 778495b87e67af1c7409a5163364d91afaf22bb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 23:57:40 -0800 Subject: [PATCH 290/551] chore(deps): bump once_cell from 1.16.0 to 1.17.0 (#270) Bumps [once_cell](https://github.com/matklad/once_cell) from 1.16.0 to 1.17.0. - [Release notes](https://github.com/matklad/once_cell/releases) - [Changelog](https://github.com/matklad/once_cell/blob/master/CHANGELOG.md) - [Commits](https://github.com/matklad/once_cell/compare/v1.16.0...v1.17.0) --- updated-dependencies: - dependency-name: once_cell dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d160cfb0..c516a3b1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -628,9 +628,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "parking_lot" From f103de06ded58eb0a93927e4e8480eb12cc25a96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 23:57:53 -0800 Subject: [PATCH 291/551] chore(deps): bump serde_derive from 1.0.151 to 1.0.152 (#269) Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.151 to 1.0.152. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.151...v1.0.152) --- updated-dependencies: - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c516a3b1a..cc2c7907e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -904,9 +904,9 @@ checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" [[package]] name = "serde_derive" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", From f9188117b9169d32cd32251f544013074365e4eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 23:58:01 -0800 Subject: [PATCH 292/551] chore(deps): bump serde from 1.0.151 to 1.0.152 (#268) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.151 to 1.0.152. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.151...v1.0.152) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc2c7907e..7e93d1d6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -898,9 +898,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" [[package]] name = "serde_derive" From b2bd987659f62fc136a45e00b35876de76d5b286 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 8 Jan 2023 20:53:28 -0800 Subject: [PATCH 293/551] chore(deps): bump async-trait from 0.1.60 to 0.1.61 (#278) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.60 to 0.1.61. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/compare/0.1.60...0.1.61) --- updated-dependencies: - dependency-name: async-trait dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e93d1d6c..85fc9731a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-trait" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" +checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" dependencies = [ "proc-macro2", "quote", From 997296659e916486e874a7be29f4007540e70435 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 8 Jan 2023 20:53:39 -0800 Subject: [PATCH 294/551] chore(deps): bump base64 from 0.20.0 to 0.21.0 (#279) Bumps [base64](https://github.com/marshallpierce/rust-base64) from 0.20.0 to 0.21.0. - [Release notes](https://github.com/marshallpierce/rust-base64/releases) - [Changelog](https://github.com/marshallpierce/rust-base64/blob/master/RELEASE-NOTES.md) - [Commits](https://github.com/marshallpierce/rust-base64/compare/v0.20.0...v0.21.0) --- updated-dependencies: - dependency-name: base64 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85fc9731a..ca0011afb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,9 +51,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "bb8" @@ -661,7 +661,7 @@ version = "0.6.0-alpha1" dependencies = [ "arc-swap", "async-trait", - "base64 0.20.0", + "base64 0.21.0", "bb8", "bytes", "chrono", diff --git a/Cargo.toml b/Cargo.toml index aa853ad92..3123a6aa2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ env_logger = "0.10" parking_lot = "0.12.1" hmac = "0.12" sha2 = "0.10" -base64 = "0.20" +base64 = "0.21" stringprep = "0.1" tokio-rustls = "0.23" rustls-pemfile = "1" From 4617f1159bc157e624452918d1214688a60002a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 09:32:54 -0800 Subject: [PATCH 295/551] chore(deps): bump regex from 1.7.0 to 1.7.1 (#280) Bumps [regex](https://github.com/rust-lang/regex) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.7.0...1.7.1) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca0011afb..ccd980378 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -809,9 +809,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", From f6e8b66e1b2c325e617cb8693fc9b7d71dbe3135 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Tue, 10 Jan 2023 20:11:40 -0500 Subject: [PATCH 296/551] Update cargo lock file (#281) Major change being updating tokio to latest version to 1.24 which has CPU performance improvements --- Cargo.lock | 361 ++++++++++++++++++++++------------------------------- 1 file changed, 149 insertions(+), 212 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ccd980378..bc1a91ef2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "aho-corasick" -version = "0.7.18" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] @@ -39,15 +39,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" - -[[package]] -name = "base64" -version = "0.13.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base64" @@ -76,18 +70,18 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" -version = "0.10.0" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d36a02058e76b040de25a4464ba1c80935655595b661505c8b39b664828b95" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" dependencies = [ "generic-array", ] [[package]] name = "bumpalo" -version = "3.10.0" +version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "bytes" @@ -97,9 +91,9 @@ checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "cc" -version = "1.0.73" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" [[package]] name = "cfg-if" @@ -140,18 +134,18 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.1" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" dependencies = [ "libc", ] [[package]] name = "crypto-common" -version = "0.1.3" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", @@ -159,9 +153,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.80" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a" +checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" dependencies = [ "cc", "cxxbridge-flags", @@ -171,9 +165,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.80" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827" +checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" dependencies = [ "cc", "codespan-reporting", @@ -186,15 +180,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.80" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a" +checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" [[package]] name = "cxxbridge-macro" -version = "1.0.80" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7" +checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" dependencies = [ "proc-macro2", "quote", @@ -203,9 +197,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ "block-buffer", "crypto-common", @@ -266,36 +260,36 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" [[package]] name = "futures-channel" -version = "0.3.19" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3dda0b6588335f360afc675d0564c17a77a2bda81ca178a4b6081bd86c7f0b" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.19" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" [[package]] name = "futures-sink" -version = "0.3.21" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" [[package]] name = "futures-task" -version = "0.3.19" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee7c6485c30167ce4dfb83ac568a849fe53274c831081476ee13e0dce1aad72" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" [[package]] name = "futures-util" -version = "0.3.19" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b5cf40b47a271f77a8b1bec03ca09044d99d2372c0de244e66430761127164" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-channel", "futures-core", @@ -307,9 +301,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" dependencies = [ "typenum", "version_check", @@ -317,20 +311,20 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.4" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", - "wasi 0.10.0+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] name = "h2" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" dependencies = [ "bytes", "fnv", @@ -459,9 +453,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", @@ -469,31 +463,31 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d367024b3f3414d8e01f437f704f41a9f64ab36f9067fa73e526ad4c763c87" +checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "is-terminal" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae5bc6e2eb41c9def29a3e0f1306382807764b9b53112030eff57435667352d" +checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" dependencies = [ "hermit-abi", "io-lifetimes", "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "itoa" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "jemalloc-sys" @@ -518,46 +512,41 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.58" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - [[package]] name = "libc" -version = "0.2.137" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "link-cplusplus" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" dependencies = [ "cc", ] [[package]] name = "linux-raw-sys" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ + "autocfg", "scopeguard", ] @@ -581,9 +570,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "mio" @@ -594,14 +583,14 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -609,9 +598,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -644,15 +633,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys 0.36.1", + "windows-sys", ] [[package]] @@ -661,7 +650,7 @@ version = "0.6.0-alpha1" dependencies = [ "arc-swap", "async-trait", - "base64 0.21.0", + "base64", "bb8", "bytes", "chrono", @@ -734,9 +723,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -746,24 +735,24 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.15" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -791,18 +780,18 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ "bitflags", ] @@ -820,9 +809,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "ring" @@ -841,23 +830,23 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.3" +version = "0.36.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b1fbb4dfc4eb1d390c02df47760bb19a84bb80b301ecc947ab5406394d8223e" +checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "rustls" -version = "0.20.6" +version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" +checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" dependencies = [ "log", "ring", @@ -867,11 +856,11 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64 0.13.1", + "base64", ] [[package]] @@ -882,9 +871,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" [[package]] name = "sct" @@ -952,15 +941,18 @@ checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] name = "slab" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] [[package]] name = "smallvec" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" @@ -1005,9 +997,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.104" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -1016,18 +1008,18 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -1051,29 +1043,29 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.19.2" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ + "autocfg", "bytes", "libc", "memchr", "mio", "num_cpus", - "once_cell", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "winapi", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "1.7.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", @@ -1093,9 +1085,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" +checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" dependencies = [ "bytes", "futures-core", @@ -1122,47 +1114,35 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.34" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tracing-core" -version = "0.1.26" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ - "lazy_static", + "once_cell", ] [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" @@ -1172,15 +1152,15 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" -version = "0.1.19" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] @@ -1227,9 +1207,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.81" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1237,13 +1217,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.81" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", "syn", @@ -1252,9 +1232,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.81" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1262,9 +1242,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.81" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", @@ -1275,15 +1255,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.81" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" [[package]] name = "web-sys" -version = "0.3.58" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" dependencies = [ "js-sys", "wasm-bindgen", @@ -1330,19 +1310,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -1350,12 +1317,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc", ] [[package]] @@ -1364,48 +1331,24 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" -[[package]] -name = "windows_aarch64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" -[[package]] -name = "windows_i686_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" -[[package]] -name = "windows_x86_64_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - [[package]] name = "windows_x86_64_gnu" version = "0.42.0" @@ -1418,12 +1361,6 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" -[[package]] -name = "windows_x86_64_msvc" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" - [[package]] name = "windows_x86_64_msvc" version = "0.42.0" From 25b29197bb507c86440d933f4ac3ffd8ba15e819 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Mon, 16 Jan 2023 23:22:06 -0500 Subject: [PATCH 297/551] Write messages directly onto message buffer instead of allocating on own buffer (#283) * initial commit * comment * fmt --- src/messages.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/messages.rs b/src/messages.rs index 826508ee5..e83155036 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -493,9 +493,21 @@ where } }; - let mut buf = vec![0u8; len as usize - 4]; + let mut bytes = BytesMut::with_capacity(len as usize + 1); + + bytes.put_u8(code); + bytes.put_i32(len); + + bytes.resize(bytes.len() + len as usize - mem::size_of::(), b'0'); - match stream.read_exact(&mut buf).await { + match stream + .read_exact( + &mut bytes[mem::size_of::() + mem::size_of::() + ..mem::size_of::() + mem::size_of::() + len as usize + - mem::size_of::()], + ) + .await + { Ok(_) => (), Err(_) => { return Err(Error::SocketError(format!( @@ -505,12 +517,6 @@ where } }; - let mut bytes = BytesMut::with_capacity(len as usize + 1); - - bytes.put_u8(code); - bytes.put_i32(len); - bytes.put_slice(&buf); - Ok(bytes) } From 74aec62d7bd218a8a41770def33dd80b2dabb559 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 17 Jan 2023 06:52:18 -0600 Subject: [PATCH 298/551] Introduce least-outstanding-connections load balancing (#282) Least outstanding connections load balancing can improve the load distribution between instances but for Pgcat it may also improve handling slow replicas that don't go completely down. With LoC, traffic will quickly move away from the slow replica without waiting for the replica to be banned. If all replicas slow down equally (due to a bad query that is hitting all replicas), the algorithm will degenerate to Random Load Balancing (which is what we had in Pgcat until today). This may also allow Pgcat to accommodate pools with differently-sized replicas. --- src/config.rs | 36 +++++++++- src/pool.rs | 32 ++++++++- src/query_router.rs | 1 + tests/ruby/helpers/pgcat_helper.rb | 9 ++- tests/ruby/load_balancing_spec.rb | 106 ++++++++++++++++++++++++++++- 5 files changed, 177 insertions(+), 7 deletions(-) diff --git a/src/config.rs b/src/config.rs index e8be9477e..219f0debc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -264,7 +264,6 @@ pub enum PoolMode { #[serde(alias = "session", alias = "Session")] Session, } - impl ToString for PoolMode { fn to_string(&self) -> String { match *self { @@ -274,11 +273,33 @@ impl ToString for PoolMode { } } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Copy, Hash)] +pub enum LoadBalancingMode { + #[serde(alias = "random", alias = "Random")] + Random, + + #[serde(alias = "loc", alias = "LOC", alias = "least_outstanding_connections")] + LeastOutstandingConnections, +} +impl ToString for LoadBalancingMode { + fn to_string(&self) -> String { + match *self { + LoadBalancingMode::Random => "random".to_string(), + LoadBalancingMode::LeastOutstandingConnections => { + "least_outstanding_connections".to_string() + } + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)] pub struct Pool { #[serde(default = "Pool::default_pool_mode")] pub pool_mode: PoolMode, + #[serde(default = "Pool::default_load_balancing_mode")] + pub load_balancing_mode: LoadBalancingMode, + pub default_role: String, #[serde(default)] // False @@ -305,6 +326,10 @@ impl Pool { PoolMode::Transaction } + pub fn default_load_balancing_mode() -> LoadBalancingMode { + LoadBalancingMode::Random + } + pub fn default_automatic_sharding_key() -> Option { None } @@ -345,6 +370,7 @@ impl Default for Pool { fn default() -> Pool { Pool { pool_mode: Self::default_pool_mode(), + load_balancing_mode: Self::default_load_balancing_mode(), shards: BTreeMap::from([(String::from("1"), Shard::default())]), users: BTreeMap::default(), default_role: String::from("any"), @@ -471,6 +497,10 @@ impl From<&Config> for std::collections::HashMap { format!("pools.{}.pool_mode", pool_name), pool.pool_mode.to_string(), ), + ( + format!("pools.{}.load_balancing_mode", pool_name), + pool.load_balancing_mode.to_string(), + ), ( format!("pools.{}.primary_reads_enabled", pool_name), pool.primary_reads_enabled.to_string(), @@ -594,6 +624,10 @@ impl Config { "[pool: {}] Pool mode: {:?}", pool_name, pool_config.pool_mode ); + info!( + "[pool: {}] Load Balancing mode: {:?}", + pool_name, pool_config.load_balancing_mode + ); let connect_timeout = match pool_config.connect_timeout { Some(connect_timeout) => connect_timeout, None => self.general.connect_timeout, diff --git a/src/pool.rs b/src/pool.rs index 94f696271..82720aaa9 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -12,7 +12,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Instant; -use crate::config::{get_config, Address, General, PoolMode, Role, User}; +use crate::config::{get_config, Address, General, LoadBalancingMode, PoolMode, Role, User}; use crate::errors::Error; use crate::server::Server; @@ -62,6 +62,9 @@ pub struct PoolSettings { /// Transaction or Session. pub pool_mode: PoolMode, + /// Random or LeastOutstandingConnections. + pub load_balancing_mode: LoadBalancingMode, + // Number of shards. pub shards: usize, @@ -94,6 +97,7 @@ impl Default for PoolSettings { fn default() -> PoolSettings { PoolSettings { pool_mode: PoolMode::Transaction, + load_balancing_mode: LoadBalancingMode::Random, shards: 1, user: User::default(), default_role: None, @@ -257,6 +261,7 @@ impl ConnectionPool { server_info: BytesMut::new(), settings: PoolSettings { pool_mode: pool_config.pool_mode, + load_balancing_mode: pool_config.load_balancing_mode, // shards: pool_config.shards.clone(), shards: shard_ids.len(), user: user.clone(), @@ -356,8 +361,17 @@ impl ConnectionPool { .filter(|address| address.role == role) .collect(); - // Random load balancing + // We shuffle even if least_outstanding_queries is used to avoid imbalance + // in cases where all candidates have more or less the same number of outstanding + // queries candidates.shuffle(&mut thread_rng()); + if self.settings.load_balancing_mode == LoadBalancingMode::LeastOutstandingConnections { + candidates.sort_by(|a, b| { + self.busy_connection_count(b) + .partial_cmp(&self.busy_connection_count(a)) + .unwrap() + }); + } while !candidates.is_empty() { // Get the next candidate @@ -565,6 +579,20 @@ impl ConnectionPool { pub fn server_info(&self) -> BytesMut { self.server_info.clone() } + + fn busy_connection_count(&self, address: &Address) -> u32 { + let state = self.pool_state(address.shard, address.address_index); + let idle = state.idle_connections; + let provisioned = state.connections; + + if idle > provisioned { + // Unlikely but avoids an overflow panic if this ever happens + return 0; + } + let busy = provisioned - idle; + debug!("{:?} has {:?} busy connections", address, busy); + return busy; + } } /// Wrapper for the bb8 connection pool. diff --git a/src/query_router.rs b/src/query_router.rs index 509057166..03f460191 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -768,6 +768,7 @@ mod test { let pool_settings = PoolSettings { pool_mode: PoolMode::Transaction, + load_balancing_mode: crate::config::LoadBalancingMode::Random, shards: 2, user: crate::config::User::default(), default_role: Some(Role::Replica), diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index 55847ed65..ffa60953b 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -5,7 +5,7 @@ module Helpers module Pgcat - def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction") + def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random") user = { "password" => "sharding_user", "pool_size" => pool_size, @@ -23,6 +23,7 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction") "#{pool_name}" => { "default_role" => "any", "pool_mode" => pool_mode, + "load_balancing_mode" => lb_mode, "primary_reads_enabled" => false, "query_parser_enabled" => false, "sharding_function" => "pg_bigint_hash", @@ -46,7 +47,7 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction") end end - def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction") + def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random") user = { "password" => "sharding_user", "pool_size" => pool_size, @@ -64,6 +65,7 @@ def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction") "#{pool_name}" => { "default_role" => "primary", "pool_mode" => pool_mode, + "load_balancing_mode" => lb_mode, "primary_reads_enabled" => false, "query_parser_enabled" => false, "sharding_function" => "pg_bigint_hash", @@ -90,7 +92,7 @@ def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction") end end - def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction") + def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random") user = { "password" => "sharding_user", "pool_size" => pool_size, @@ -111,6 +113,7 @@ def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction") "#{pool_name}" => { "default_role" => "any", "pool_mode" => pool_mode, + "load_balancing_mode" => lb_mode, "primary_reads_enabled" => false, "query_parser_enabled" => false, "sharding_function" => "pg_bigint_hash", diff --git a/tests/ruby/load_balancing_spec.rb b/tests/ruby/load_balancing_spec.rb index bd98a831c..8be066df4 100644 --- a/tests/ruby/load_balancing_spec.rb +++ b/tests/ruby/load_balancing_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'spec_helper' -describe "Load Balancing" do +describe "Random Load Balancing" do let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5) } after do processes.all_databases.map(&:reset) @@ -59,3 +59,107 @@ end end +describe "Least Outstanding Queries Load Balancing" do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 1, "transaction", "loc") } + after do + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + + context "under homogenous load" do + it "balances query volume between all instances" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + + query_count = QUERY_COUNT + expected_share = query_count / processes.all_databases.count + failed_count = 0 + + query_count.times do + conn.async_exec("SELECT 1 + 2") + rescue + failed_count += 1 + end + + expect(failed_count).to eq(0) + processes.all_databases.map(&:count_select_1_plus_2).each do |instance_share| + expect(instance_share).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share) + end + end + end + + context "under heterogeneous load" do + it "balances query volume between all instances based on how busy they are" do + slow_query_count = 2 + threads = Array.new(slow_query_count) do + Thread.new do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SELECT pg_sleep(1)") + end + end + + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + + query_count = QUERY_COUNT + expected_share = query_count / (processes.all_databases.count - slow_query_count) + failed_count = 0 + + query_count.times do + conn.async_exec("SELECT 1 + 2") + rescue + failed_count += 1 + end + + expect(failed_count).to eq(0) + # Under LOQ, we expect replicas running the slow pg_sleep + # to get no selects + expect( + processes. + all_databases. + map(&:count_select_1_plus_2). + count { |instance_share| instance_share == 0 } + ).to eq(slow_query_count) + + # We also expect the quick queries to be spread across + # the idle servers only + processes. + all_databases. + map(&:count_select_1_plus_2). + reject { |instance_share| instance_share == 0 }. + each do |instance_share| + expect(instance_share).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share) + end + + threads.map(&:join) + end + end + + context "when some replicas are down" do + it "balances query volume between working instances" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + expected_share = QUERY_COUNT / (processes.all_databases.count - 2) + failed_count = 0 + + processes[:replicas][0].take_down do + processes[:replicas][1].take_down do + QUERY_COUNT.times do + conn.async_exec("SELECT 1 + 2") + rescue + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + failed_count += 1 + end + end + end + + expect(failed_count).to eq(2) + processes.all_databases.each do |instance| + queries_routed = instance.count_select_1_plus_2 + if processes.replicas[0..1].include?(instance) + expect(queries_routed).to eq(0) + else + expect(queries_routed).to be_within(expected_share * MARGIN_OF_ERROR).of(expected_share) + end + end + end + end +end + From f1274d4f597de5a48ac689065b3ee3f31737efb2 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Tue, 17 Jan 2023 20:39:55 -0500 Subject: [PATCH 299/551] Buffer client CopyData messages (#284) Buffers CopyData messages and removes buffer clone for the sync message --- src/admin.rs | 18 +++++++++--------- src/client.rs | 46 ++++++++++++++++++++++++++++------------------ src/messages.rs | 10 +++++----- src/server.rs | 4 ++-- 4 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 4460f9821..5879114ac 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -171,7 +171,7 @@ where res.put_i32(5); res.put_u8(b'I'); - write_all_half(stream, res).await + write_all_half(stream, &res).await } /// Show PgCat version. @@ -189,7 +189,7 @@ where res.put_i32(5); res.put_u8(b'I'); - write_all_half(stream, res).await + write_all_half(stream, &res).await } /// Show utilization of connection pools for each shard and replicas. @@ -250,7 +250,7 @@ where res.put_i32(5); res.put_u8(b'I'); - write_all_half(stream, res).await + write_all_half(stream, &res).await } /// Show shards and replicas. @@ -317,7 +317,7 @@ where res.put_i32(5); res.put_u8(b'I'); - write_all_half(stream, res).await + write_all_half(stream, &res).await } /// Ignore any SET commands the client sends. @@ -349,7 +349,7 @@ where res.put_i32(5); res.put_u8(b'I'); - write_all_half(stream, res).await + write_all_half(stream, &res).await } /// Shows current configuration. @@ -395,7 +395,7 @@ where res.put_i32(5); res.put_u8(b'I'); - write_all_half(stream, res).await + write_all_half(stream, &res).await } /// Show shard and replicas statistics. @@ -455,7 +455,7 @@ where res.put_i32(5); res.put_u8(b'I'); - write_all_half(stream, res).await + write_all_half(stream, &res).await } /// Show currently connected clients @@ -505,7 +505,7 @@ where res.put_i32(5); res.put_u8(b'I'); - write_all_half(stream, res).await + write_all_half(stream, &res).await } /// Show currently connected servers @@ -559,5 +559,5 @@ where res.put_i32(5); res.put_u8(b'I'); - write_all_half(stream, res).await + write_all_half(stream, &res).await } diff --git a/src/client.rs b/src/client.rs index b55906b2b..cfe12c0e5 100644 --- a/src/client.rs +++ b/src/client.rs @@ -861,7 +861,7 @@ where 'Q' => { debug!("Sending query to server"); - self.send_and_receive_loop(code, message, server, &address, &pool) + self.send_and_receive_loop(code, Some(&message), server, &address, &pool) .await?; if !server.in_transaction() { @@ -931,14 +931,8 @@ where } } - self.send_and_receive_loop( - code, - self.buffer.clone(), - server, - &address, - &pool, - ) - .await?; + self.send_and_receive_loop(code, None, server, &address, &pool) + .await?; self.buffer.clear(); @@ -955,21 +949,32 @@ where // CopyData 'd' => { - // Forward the data to the server, - // don't buffer it since it can be rather large. - self.send_server_message(server, message, &address, &pool) - .await?; + self.buffer.put(&message[..]); + + // Want to limit buffer size + if self.buffer.len() > 8196 { + // Forward the data to the server, + self.send_server_message(server, &self.buffer, &address, &pool) + .await?; + self.buffer.clear(); + } } // CopyDone or CopyFail // Copy is done, successfully or not. 'c' | 'f' => { - self.send_server_message(server, message, &address, &pool) + // We may already have some copy data in the buffer, add this message to buffer + self.buffer.put(&message[..]); + + self.send_server_message(server, &self.buffer, &address, &pool) .await?; + // Clear the buffer + self.buffer.clear(); + let response = self.receive_server_message(server, &address, &pool).await?; - match write_all_half(&mut self.write, response).await { + match write_all_half(&mut self.write, &response).await { Ok(_) => (), Err(err) => { server.mark_bad(); @@ -1016,13 +1021,18 @@ where async fn send_and_receive_loop( &mut self, code: char, - message: BytesMut, + message: Option<&BytesMut>, server: &mut Server, address: &Address, pool: &ConnectionPool, ) -> Result<(), Error> { debug!("Sending {} to server", code); + let message = match message { + Some(message) => message, + None => &self.buffer, + }; + self.send_server_message(server, message, address, pool) .await?; @@ -1032,7 +1042,7 @@ where loop { let response = self.receive_server_message(server, address, pool).await?; - match write_all_half(&mut self.write, response).await { + match write_all_half(&mut self.write, &response).await { Ok(_) => (), Err(err) => { server.mark_bad(); @@ -1058,7 +1068,7 @@ where async fn send_server_message( &self, server: &mut Server, - message: BytesMut, + message: &BytesMut, address: &Address, pool: &ConnectionPool, ) -> Result<(), Error> { diff --git a/src/messages.rs b/src/messages.rs index e83155036..45a827c82 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -258,7 +258,7 @@ where res.put_i32(len); res.put_slice(&set_complete[..]); - write_all_half(stream, res).await?; + write_all_half(stream, &res).await?; ready_for_query(stream).await } @@ -308,7 +308,7 @@ where res.put_i32(error.len() as i32 + 4); res.put(error); - write_all_half(stream, res).await + write_all_half(stream, &res).await } pub async fn wrong_password(stream: &mut S, user: &str) -> Result<(), Error> @@ -370,7 +370,7 @@ where // CommandComplete res.put(command_complete("SELECT 1")); - write_all_half(stream, res).await?; + write_all_half(stream, &res).await?; ready_for_query(stream).await } @@ -459,11 +459,11 @@ where } /// Write all the data in the buffer to the TcpStream, write owned half (see mpsc). -pub async fn write_all_half(stream: &mut S, buf: BytesMut) -> Result<(), Error> +pub async fn write_all_half(stream: &mut S, buf: &BytesMut) -> Result<(), Error> where S: tokio::io::AsyncWrite + std::marker::Unpin, { - match stream.write_all(&buf).await { + match stream.write_all(buf).await { Ok(_) => Ok(()), Err(_) => return Err(Error::SocketError(format!("Error writing to socket"))), } diff --git a/src/server.rs b/src/server.rs index 05a3b770e..f2a6d387d 100644 --- a/src/server.rs +++ b/src/server.rs @@ -381,7 +381,7 @@ impl Server { } /// Send messages to the server from the client. - pub async fn send(&mut self, messages: BytesMut) -> Result<(), Error> { + pub async fn send(&mut self, messages: &BytesMut) -> Result<(), Error> { self.stats.data_sent(messages.len(), self.server_id); match write_all_half(&mut self.write, messages).await { @@ -593,7 +593,7 @@ impl Server { pub async fn query(&mut self, query: &str) -> Result<(), Error> { let query = simple_query(query); - self.send(query).await?; + self.send(&query).await?; loop { let _ = self.recv().await?; From 0e77536184e233b0204126f6172511c481b1989d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Jan 2023 08:51:07 -0800 Subject: [PATCH 300/551] chore(deps): bump tokio from 1.24.1 to 1.24.2 (#286) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.24.1 to 1.24.2. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/commits) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc1a91ef2..08a9a2159 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1043,9 +1043,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.24.1" +version = "1.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" +checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" dependencies = [ "autocfg", "bytes", From 9a8d652b0c8d7fac9a77fa36d9f5f50dbee9afe8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Jan 2023 16:56:16 -0800 Subject: [PATCH 301/551] chore(deps): bump activerecord from 7.0.3.1 to 7.0.4.1 in /tests/ruby (#287) Bumps [activerecord](https://github.com/rails/rails) from 7.0.3.1 to 7.0.4.1. - [Release notes](https://github.com/rails/rails/releases) - [Changelog](https://github.com/rails/rails/blob/v7.0.4.1/activerecord/CHANGELOG.md) - [Commits](https://github.com/rails/rails/compare/v7.0.3.1...v7.0.4.1) --- updated-dependencies: - dependency-name: activerecord dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- tests/ruby/Gemfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock index 65d8bce7a..f49468053 100644 --- a/tests/ruby/Gemfile.lock +++ b/tests/ruby/Gemfile.lock @@ -1,12 +1,12 @@ GEM remote: https://rubygems.org/ specs: - activemodel (7.0.3.1) - activesupport (= 7.0.3.1) - activerecord (7.0.3.1) - activemodel (= 7.0.3.1) - activesupport (= 7.0.3.1) - activesupport (7.0.3.1) + activemodel (7.0.4.1) + activesupport (= 7.0.4.1) + activerecord (7.0.4.1) + activemodel (= 7.0.4.1) + activesupport (= 7.0.4.1) + activesupport (7.0.4.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -14,9 +14,9 @@ GEM ast (2.4.2) concurrent-ruby (1.1.10) diff-lcs (1.5.0) - i18n (1.11.0) + i18n (1.12.0) concurrent-ruby (~> 1.0) - minitest (5.16.2) + minitest (5.17.0) parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) @@ -53,7 +53,7 @@ GEM toml (0.3.0) parslet (>= 1.8.0, < 3.0.0) toxiproxy (2.0.1) - tzinfo (2.0.4) + tzinfo (2.0.5) concurrent-ruby (~> 1.0) unicode-display_width (2.1.0) From aeefd4c6e5d6d6f996cf36ad60973e6864ffdc87 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 19 Jan 2023 05:18:08 -0600 Subject: [PATCH 302/551] Log error messages for network failures (#289) We are seeing some Error reading message code from socket error messages, we want to get more context so this PR logs the actual error reported. --- src/messages.rs | 36 ++++++++++++++++++++----------- tests/ruby/load_balancing_spec.rb | 2 +- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/messages.rs b/src/messages.rs index 45a827c82..8e49653ac 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -136,9 +136,10 @@ pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Resu match stream.write_all(&startup).await { Ok(_) => Ok(()), - Err(_) => { + Err(err) => { return Err(Error::SocketError(format!( - "Error writing startup to server socket" + "Error writing startup to server socket - Error: {:?}", + err ))) } } @@ -454,7 +455,12 @@ where { match stream.write_all(&buf).await { Ok(_) => Ok(()), - Err(_) => return Err(Error::SocketError(format!("Error writing to socket"))), + Err(err) => { + return Err(Error::SocketError(format!( + "Error writing to socket - Error: {:?}", + err + ))) + } } } @@ -465,7 +471,12 @@ where { match stream.write_all(buf).await { Ok(_) => Ok(()), - Err(_) => return Err(Error::SocketError(format!("Error writing to socket"))), + Err(err) => { + return Err(Error::SocketError(format!( + "Error writing to socket - Error: {:?}", + err + ))) + } } } @@ -476,19 +487,20 @@ where { let code = match stream.read_u8().await { Ok(code) => code, - Err(_) => { + Err(err) => { return Err(Error::SocketError(format!( - "Error reading message code from socket" + "Error reading message code from socket - Error {:?}", + err ))) } }; let len = match stream.read_i32().await { Ok(len) => len, - Err(_) => { + Err(err) => { return Err(Error::SocketError(format!( - "Error reading message len from socket, code: {:?}", - code + "Error reading message len from socket - Code: {:?}, Error: {:?}", + code, err ))) } }; @@ -509,10 +521,10 @@ where .await { Ok(_) => (), - Err(_) => { + Err(err) => { return Err(Error::SocketError(format!( - "Error reading message from socket, code: {:?}", - code + "Error reading message from socket - Code: {:?}, Error: {:?}", + code, err ))) } }; diff --git a/tests/ruby/load_balancing_spec.rb b/tests/ruby/load_balancing_spec.rb index 8be066df4..fccf0a859 100644 --- a/tests/ruby/load_balancing_spec.rb +++ b/tests/ruby/load_balancing_spec.rb @@ -93,7 +93,7 @@ threads = Array.new(slow_query_count) do Thread.new do conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) - conn.async_exec("SELECT pg_sleep(1)") + conn.async_exec("BEGIN") end end From 32746aa9c6a85a6e4ebe0b8aafde8ad489dab873 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 19 Jan 2023 10:19:49 -0500 Subject: [PATCH 303/551] Removes message cloning operation required for query router (#285) * Removes message cloning operation required for query router * fmt * flakey? * ? --- src/client.rs | 4 +- src/errors.rs | 1 + src/messages.rs | 17 ++++++++ src/query_router.rs | 94 ++++++++++++++++++++++----------------------- 4 files changed, 65 insertions(+), 51 deletions(-) diff --git a/src/client.rs b/src/client.rs index cfe12c0e5..15fe21d91 100644 --- a/src/client.rs +++ b/src/client.rs @@ -693,11 +693,11 @@ where let current_shard = query_router.shard(); // Handle all custom protocol commands, if any. - match query_router.try_execute_command(message.clone()) { + match query_router.try_execute_command(&message) { // Normal query, not a custom command. None => { if query_router.query_parser_enabled() { - query_router.infer(message.clone()); + query_router.infer(&message); } } diff --git a/src/errors.rs b/src/errors.rs index 7789a8a77..4ac23a855 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -13,4 +13,5 @@ pub enum Error { TlsError, StatementTimeout, ShuttingDown, + ParseBytesError(String), } diff --git a/src/messages.rs b/src/messages.rs index 8e49653ac..e7c36747a 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -7,6 +7,7 @@ use tokio::net::TcpStream; use crate::errors::Error; use std::collections::HashMap; +use std::io::{BufRead, Cursor}; use std::mem; /// Postgres data type mappings @@ -548,3 +549,19 @@ pub fn server_parameter_message(key: &str, value: &str) -> BytesMut { server_info } + +pub trait BytesMutReader { + fn read_string(&mut self) -> Result; +} + +impl BytesMutReader for Cursor<&BytesMut> { + /// Should only be used when reading strings from the message protocol. + /// Can be used to read multiple strings from the same message which are separated by the null byte + fn read_string(&mut self) -> Result { + let mut buf = vec![]; + match self.read_until(b'\0', &mut buf) { + Ok(_) => Ok(String::from_utf8_lossy(&buf[..buf.len() - 1]).to_string()), + Err(err) => return Err(Error::ParseBytesError(err.to_string())), + } + } +} diff --git a/src/query_router.rs b/src/query_router.rs index 03f460191..9f9dcd76e 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -10,10 +10,12 @@ use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; use crate::config::Role; +use crate::messages::BytesMutReader; use crate::pool::PoolSettings; use crate::sharding::Sharder; use std::collections::BTreeSet; +use std::io::Cursor; /// Regexes used to parse custom commands. const CUSTOM_SQL_REGEXES: [&str; 7] = [ @@ -107,16 +109,18 @@ impl QueryRouter { } /// Try to parse a command and execute it. - pub fn try_execute_command(&mut self, mut buf: BytesMut) -> Option<(Command, String)> { - let code = buf.get_u8() as char; + pub fn try_execute_command(&mut self, message_buffer: &BytesMut) -> Option<(Command, String)> { + let mut message_cursor = Cursor::new(message_buffer); + + let code = message_cursor.get_u8() as char; // Only simple protocol supported for commands. if code != 'Q' { return None; } - let len = buf.get_i32() as usize; - let query = String::from_utf8_lossy(&buf[..len - 5]).to_string(); // Ignore the terminating NULL. + let _len = message_cursor.get_i32() as usize; + let query = message_cursor.read_string().unwrap(); let regex_set = match CUSTOM_SQL_REGEX_SET.get() { Some(regex_set) => regex_set, @@ -256,37 +260,29 @@ impl QueryRouter { } /// Try to infer which server to connect to based on the contents of the query. - pub fn infer(&mut self, mut buf: BytesMut) -> bool { + pub fn infer(&mut self, message_buffer: &BytesMut) -> bool { debug!("Inferring role"); - let code = buf.get_u8() as char; - let len = buf.get_i32() as usize; + let mut message_cursor = Cursor::new(message_buffer); + + let code = message_cursor.get_u8() as char; + let _len = message_cursor.get_i32() as usize; let query = match code { // Query 'Q' => { - let query = String::from_utf8_lossy(&buf[..len - 5]).to_string(); + let query = message_cursor.read_string().unwrap(); debug!("Query: '{}'", query); query } // Parse (prepared statement) 'P' => { - let mut start = 0; - - // Skip the name of the prepared statement. - while buf[start] != 0 && start < buf.len() { - start += 1; - } - start += 1; // Skip terminating null - - // Find the end of the prepared stmt (\0) - let mut end = start; - while buf[end] != 0 && end < buf.len() { - end += 1; - } + // Reads statement name + message_cursor.read_string().unwrap(); - let query = String::from_utf8_lossy(&buf[start..end]).to_string(); + // Reads query string + let query = message_cursor.read_string().unwrap(); debug!("Prepared statement: '{}'", query); @@ -519,10 +515,10 @@ mod test { fn test_infer_replica() { QueryRouter::setup(); let mut qr = QueryRouter::new(); - assert!(qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")) != None); + assert!(qr.try_execute_command(&simple_query("SET SERVER ROLE TO 'auto'")) != None); assert!(qr.query_parser_enabled()); - assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); + assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO off")) != None); let queries = vec![ simple_query("SELECT * FROM items WHERE id = 5"), @@ -534,7 +530,7 @@ mod test { for query in queries { // It's a recognized query - assert!(qr.infer(query)); + assert!(qr.infer(&query)); assert_eq!(qr.role(), Some(Role::Replica)); } } @@ -553,7 +549,7 @@ mod test { for query in queries { // It's a recognized query - assert!(qr.infer(query)); + assert!(qr.infer(&query)); assert_eq!(qr.role(), Some(Role::Primary)); } } @@ -563,9 +559,9 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); let query = simple_query("SELECT * FROM items WHERE id = 5"); - assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO on")) != None); + assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO on")) != None); - assert!(qr.infer(query)); + assert!(qr.infer(&query)); assert_eq!(qr.role(), None); } @@ -573,8 +569,8 @@ mod test { fn test_infer_parse_prepared() { QueryRouter::setup(); let mut qr = QueryRouter::new(); - qr.try_execute_command(simple_query("SET SERVER ROLE TO 'auto'")); - assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); + qr.try_execute_command(&simple_query("SET SERVER ROLE TO 'auto'")); + assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO off")) != None); let prepared_stmt = BytesMut::from( &b"WITH t AS (SELECT * FROM items WHERE name = $1) SELECT * FROM t WHERE id = $2\0"[..], @@ -585,7 +581,7 @@ mod test { res.put(prepared_stmt); res.put_i16(0); - assert!(qr.infer(res)); + assert!(qr.infer(&res)); assert_eq!(qr.role(), Some(Role::Replica)); } @@ -668,7 +664,7 @@ mod test { // SetShardingKey let query = simple_query("SET SHARDING KEY TO 13"); assert_eq!( - qr.try_execute_command(query), + qr.try_execute_command(&query), Some((Command::SetShardingKey, String::from("0"))) ); assert_eq!(qr.shard(), 0); @@ -676,7 +672,7 @@ mod test { // SetShard let query = simple_query("SET SHARD TO '1'"); assert_eq!( - qr.try_execute_command(query), + qr.try_execute_command(&query), Some((Command::SetShard, String::from("1"))) ); assert_eq!(qr.shard(), 1); @@ -684,7 +680,7 @@ mod test { // ShowShard let query = simple_query("SHOW SHARD"); assert_eq!( - qr.try_execute_command(query), + qr.try_execute_command(&query), Some((Command::ShowShard, String::from("1"))) ); @@ -702,7 +698,7 @@ mod test { for (idx, role) in roles.iter().enumerate() { let query = simple_query(&format!("SET SERVER ROLE TO '{}'", role)); assert_eq!( - qr.try_execute_command(query), + qr.try_execute_command(&query), Some((Command::SetServerRole, String::from(*role))) ); assert_eq!(qr.role(), verify_roles[idx],); @@ -711,7 +707,7 @@ mod test { // ShowServerRole let query = simple_query("SHOW SERVER ROLE"); assert_eq!( - qr.try_execute_command(query), + qr.try_execute_command(&query), Some((Command::ShowServerRole, String::from(*role))) ); } @@ -721,14 +717,14 @@ mod test { for (idx, primary_reads) in primary_reads.iter().enumerate() { assert_eq!( - qr.try_execute_command(simple_query(&format!( + qr.try_execute_command(&simple_query(&format!( "SET PRIMARY READS TO {}", primary_reads ))), Some((Command::SetPrimaryReads, String::from(*primary_reads))) ); assert_eq!( - qr.try_execute_command(simple_query("SHOW PRIMARY READS")), + qr.try_execute_command(&simple_query("SHOW PRIMARY READS")), Some(( Command::ShowPrimaryReads, String::from(primary_reads_enabled[idx]) @@ -742,23 +738,23 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); let query = simple_query("SET SERVER ROLE TO 'auto'"); - assert!(qr.try_execute_command(simple_query("SET PRIMARY READS TO off")) != None); + assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO off")) != None); - assert!(qr.try_execute_command(query) != None); + assert!(qr.try_execute_command(&query) != None); assert!(qr.query_parser_enabled()); assert_eq!(qr.role(), None); let query = simple_query("INSERT INTO test_table VALUES (1)"); - assert!(qr.infer(query)); + assert!(qr.infer(&query)); assert_eq!(qr.role(), Some(Role::Primary)); let query = simple_query("SELECT * FROM test_table"); - assert!(qr.infer(query)); + assert!(qr.infer(&query)); assert_eq!(qr.role(), Some(Role::Replica)); assert!(qr.query_parser_enabled()); let query = simple_query("SET SERVER ROLE TO 'default'"); - assert!(qr.try_execute_command(query) != None); + assert!(qr.try_execute_command(&query) != None); assert!(!qr.query_parser_enabled()); } @@ -794,16 +790,16 @@ mod test { assert!(!qr.primary_reads_enabled()); let q1 = simple_query("SET SERVER ROLE TO 'primary'"); - assert!(qr.try_execute_command(q1) != None); + assert!(qr.try_execute_command(&q1) != None); assert_eq!(qr.active_role.unwrap(), Role::Primary); let q2 = simple_query("SET SERVER ROLE TO 'default'"); - assert!(qr.try_execute_command(q2) != None); + assert!(qr.try_execute_command(&q2) != None); assert_eq!(qr.active_role.unwrap(), pool_settings.default_role); // Here we go :) let q3 = simple_query("SELECT * FROM test WHERE id = 5 AND values IN (1, 2, 3)"); - assert!(qr.infer(q3)); + assert!(qr.infer(&q3)); assert_eq!(qr.shard(), 1); } @@ -812,13 +808,13 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); - assert!(qr.infer(simple_query("BEGIN; SELECT 1; COMMIT;"))); + assert!(qr.infer(&simple_query("BEGIN; SELECT 1; COMMIT;"))); assert_eq!(qr.role(), Role::Primary); - assert!(qr.infer(simple_query("SELECT 1; SELECT 2;"))); + assert!(qr.infer(&simple_query("SELECT 1; SELECT 2;"))); assert_eq!(qr.role(), Role::Replica); - assert!(qr.infer(simple_query( + assert!(qr.infer(&simple_query( "SELECT 123; INSERT INTO t VALUES (5); SELECT 1;" ))); assert_eq!(qr.role(), Role::Primary); From e6110c0596bd31cea2c3a511a1c0f3f9da9c0c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Thu, 19 Jan 2023 16:48:12 +0100 Subject: [PATCH 304/551] Add more metrics to prometheus endpoint (#263) This change: - Adds server metrics to prometheus endpoint. - Adds database metrics to prometheus endpoint. - Adds pools metrics to prometheus endpoint. - Change metrics name to have a prefix of (stats|pools|databases|servers). --- src/prometheus.rs | 278 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 220 insertions(+), 58 deletions(-) diff --git a/src/prometheus.rs b/src/prometheus.rs index ec011b396..e596f9f99 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -8,7 +8,7 @@ use std::net::SocketAddr; use crate::config::Address; use crate::pool::get_all_pools; -use crate::stats::get_address_stats; +use crate::stats::{get_address_stats, get_pool_stats, get_server_stats, ServerInformation}; struct MetricHelpType { help: &'static str, @@ -19,113 +19,141 @@ struct MetricHelpType { // counters only increase // gauges can arbitrarily increase or decrease static METRIC_HELP_AND_TYPES_LOOKUP: phf::Map<&'static str, MetricHelpType> = phf_map! { - "total_query_count" => MetricHelpType { + "stats_total_query_count" => MetricHelpType { help: "Number of queries sent by all clients", ty: "counter", }, - "total_query_time" => MetricHelpType { + "stats_total_query_time" => MetricHelpType { help: "Total amount of time for queries to execute", ty: "counter", }, - "total_received" => MetricHelpType { + "stats_total_received" => MetricHelpType { help: "Number of bytes received from the server", ty: "counter", }, - "total_sent" => MetricHelpType { + "stats_total_sent" => MetricHelpType { help: "Number of bytes sent to the server", ty: "counter", }, - "total_xact_count" => MetricHelpType { + "stats_total_xact_count" => MetricHelpType { help: "Total number of transactions started by the client", ty: "counter", }, - "total_xact_time" => MetricHelpType { + "stats_total_xact_time" => MetricHelpType { help: "Total amount of time for all transactions to execute", ty: "counter", }, - "total_wait_time" => MetricHelpType { + "stats_total_wait_time" => MetricHelpType { help: "Total time client waited for a server connection", ty: "counter", }, - "avg_query_count" => MetricHelpType { + "stats_avg_query_count" => MetricHelpType { help: "Average of total_query_count every 15 seconds", ty: "gauge", }, - "avg_query_time" => MetricHelpType { + "stats_avg_query_time" => MetricHelpType { help: "Average time taken for queries to execute every 15 seconds", ty: "gauge", }, - "avg_recv" => MetricHelpType { + "stats_avg_recv" => MetricHelpType { help: "Average of total_received bytes every 15 seconds", ty: "gauge", }, - "avg_sent" => MetricHelpType { + "stats_avg_sent" => MetricHelpType { help: "Average of total_sent bytes every 15 seconds", ty: "gauge", }, - "avg_errors" => MetricHelpType { + "stats_avg_errors" => MetricHelpType { help: "Average number of errors every 15 seconds", ty: "gauge", }, - "avg_xact_count" => MetricHelpType { + "stats_avg_xact_count" => MetricHelpType { help: "Average of total_xact_count every 15 seconds", ty: "gauge", }, - "avg_xact_time" => MetricHelpType { + "stats_avg_xact_time" => MetricHelpType { help: "Average of total_xact_time every 15 seconds", ty: "gauge", }, - "avg_wait_time" => MetricHelpType { + "stats_avg_wait_time" => MetricHelpType { help: "Average of total_wait_time every 15 seconds", ty: "gauge", }, - "maxwait_us" => MetricHelpType { + "pools_maxwait_us" => MetricHelpType { help: "The time a client waited for a server connection in microseconds", ty: "gauge", }, - "maxwait" => MetricHelpType { + "pools_maxwait" => MetricHelpType { help: "The time a client waited for a server connection in seconds", ty: "gauge", }, - "cl_waiting" => MetricHelpType { + "pools_cl_waiting" => MetricHelpType { help: "How many clients are waiting for a connection from the pool", ty: "gauge", }, - "cl_active" => MetricHelpType { + "pools_cl_active" => MetricHelpType { help: "How many clients are actively communicating with a server", ty: "gauge", }, - "cl_idle" => MetricHelpType { + "pools_cl_idle" => MetricHelpType { help: "How many clients are idle", ty: "gauge", }, - "sv_idle" => MetricHelpType { + "pools_sv_idle" => MetricHelpType { help: "How many server connections are idle", ty: "gauge", }, - "sv_active" => MetricHelpType { + "pools_sv_active" => MetricHelpType { help: "How many server connections are actively communicating with a client", ty: "gauge", }, - "sv_login" => MetricHelpType { + "pools_sv_login" => MetricHelpType { help: "How many server connections are currently being created", ty: "gauge", }, - "sv_tested" => MetricHelpType { + "pools_sv_tested" => MetricHelpType { help: "How many server connections are currently waiting on a health check to succeed", ty: "gauge", }, + "servers_bytes_received" => MetricHelpType { + help: "Volume in bytes of network traffic received by server", + ty: "gauge", + }, + "servers_bytes_sent" => MetricHelpType { + help: "Volume in bytes of network traffic sent by server", + ty: "gauge", + }, + "servers_transaction_count" => MetricHelpType { + help: "Number of transactions executed by server", + ty: "gauge", + }, + "servers_query_count" => MetricHelpType { + help: "Number of queries executed by server", + ty: "gauge", + }, + "servers_error_count" => MetricHelpType { + help: "Number of errors", + ty: "gauge", + }, + "databases_pool_size" => MetricHelpType { + help: "Maximum number of server connections", + ty: "gauge", + }, + "databases_current_connections" => MetricHelpType { + help: "Current number of connections for this database", + ty: "gauge", + }, }; -struct PrometheusMetric { +struct PrometheusMetric { name: String, help: String, ty: String, labels: HashMap<&'static str, String>, - value: i64, + value: Value, } -impl fmt::Display for PrometheusMetric { +impl fmt::Display for PrometheusMetric { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let formatted_labels = self .labels @@ -145,50 +173,81 @@ impl fmt::Display for PrometheusMetric { } } -impl PrometheusMetric { - fn new(address: &Address, name: &str, value: i64) -> Option { - let mut labels = HashMap::new(); - labels.insert("host", address.host.clone()); - labels.insert("shard", address.shard.to_string()); - labels.insert("role", address.role.to_string()); - labels.insert("database", address.database.to_string()); - +impl PrometheusMetric { + fn from_name( + name: &str, + value: V, + labels: HashMap<&'static str, String>, + ) -> Option> { METRIC_HELP_AND_TYPES_LOOKUP .get(name) - .map(|metric| PrometheusMetric { + .map(|metric| PrometheusMetric:: { name: name.to_owned(), help: metric.help.to_owned(), ty: metric.ty.to_owned(), - labels, value, + labels, }) } + + fn from_database_info( + address: &Address, + name: &str, + value: u32, + ) -> Option> { + let mut labels = HashMap::new(); + labels.insert("host", address.host.clone()); + labels.insert("shard", address.shard.to_string()); + labels.insert("role", address.role.to_string()); + labels.insert("pool", address.pool_name.clone()); + labels.insert("database", address.database.to_string()); + + Self::from_name(&format!("databases_{}", name), value, labels) + } + + fn from_server_info( + address: &Address, + name: &str, + value: u64, + ) -> Option> { + let mut labels = HashMap::new(); + labels.insert("host", address.host.clone()); + labels.insert("shard", address.shard.to_string()); + labels.insert("role", address.role.to_string()); + labels.insert("pool", address.pool_name.clone()); + labels.insert("database", address.database.to_string()); + + Self::from_name(&format!("servers_{}", name), value, labels) + } + + fn from_address(address: &Address, name: &str, value: i64) -> Option> { + let mut labels = HashMap::new(); + labels.insert("host", address.host.clone()); + labels.insert("shard", address.shard.to_string()); + labels.insert("pool", address.pool_name.clone()); + labels.insert("role", address.role.to_string()); + labels.insert("database", address.database.to_string()); + + Self::from_name(&format!("stats_{}", name), value, labels) + } + + fn from_pool(pool: &(String, String), name: &str, value: i64) -> Option> { + let mut labels = HashMap::new(); + labels.insert("pool", pool.0.clone()); + labels.insert("user", pool.1.clone()); + + Self::from_name(&format!("pools_{}", name), value, labels) + } } async fn prometheus_stats(request: Request) -> Result, hyper::http::Error> { match (request.method(), request.uri().path()) { (&Method::GET, "/metrics") => { - let stats: HashMap> = get_address_stats(); - let mut lines = Vec::new(); - for (_, pool) in get_all_pools() { - for shard in 0..pool.shards() { - for server in 0..pool.servers(shard) { - let address = pool.address(shard, server); - if let Some(address_stats) = stats.get(&address.id) { - for (key, value) in address_stats.iter() { - if let Some(prometheus_metric) = - PrometheusMetric::new(address, key, *value) - { - lines.push(prometheus_metric.to_string()); - } else { - warn!("Metric {} not implemented for {}", key, address.name()); - } - } - } - } - } - } + push_address_stats(&mut lines); + push_pool_stats(&mut lines); + push_server_stats(&mut lines); + push_database_stats(&mut lines); Response::builder() .header("content-type", "text/plain; version=0.0.4") @@ -200,6 +259,109 @@ async fn prometheus_stats(request: Request) -> Result, hype } } +// Adds metrics shown in a SHOW STATS admin command. +fn push_address_stats(lines: &mut Vec) { + let address_stats: HashMap> = get_address_stats(); + for (_, pool) in get_all_pools() { + for shard in 0..pool.shards() { + for server in 0..pool.servers(shard) { + let address = pool.address(shard, server); + if let Some(address_stats) = address_stats.get(&address.id) { + for (key, value) in address_stats.iter() { + if let Some(prometheus_metric) = + PrometheusMetric::::from_address(address, key, *value) + { + lines.push(prometheus_metric.to_string()); + } else { + warn!("Metric {} not implemented for {}", key, address.name()); + } + } + } + } + } + } +} + +// Adds relevant metrics shown in a SHOW POOLS admin command. +fn push_pool_stats(lines: &mut Vec) { + let pool_stats = get_pool_stats(); + for (pool, stats) in pool_stats.iter() { + for (name, value) in stats.iter() { + if let Some(prometheus_metric) = PrometheusMetric::::from_pool(pool, name, *value) + { + lines.push(prometheus_metric.to_string()); + } else { + warn!( + "Metric {} not implemented for ({},{})", + name, pool.0, pool.1 + ); + } + } + } +} + +// Adds relevant metrics shown in a SHOW DATABASES admin command. +fn push_database_stats(lines: &mut Vec) { + for (_, pool) in get_all_pools() { + let pool_config = pool.settings.clone(); + for shard in 0..pool.shards() { + for server in 0..pool.servers(shard) { + let address = pool.address(shard, server); + let pool_state = pool.pool_state(shard, server); + + let metrics = vec![ + ("pool_size", pool_config.user.pool_size), + ("current_connections", pool_state.connections), + ]; + for (key, value) in metrics { + if let Some(prometheus_metric) = + PrometheusMetric::::from_database_info(address, key, value) + { + lines.push(prometheus_metric.to_string()); + } else { + warn!("Metric {} not implemented for {}", key, address.name()); + } + } + } + } + } +} + +// Adds relevant metrics shown in a SHOW SERVERS admin command. +fn push_server_stats(lines: &mut Vec) { + let server_stats = get_server_stats(); + let mut server_stats_by_addresses = HashMap::::new(); + for (_, info) in server_stats { + server_stats_by_addresses.insert(info.address_name.clone(), info); + } + + for (_, pool) in get_all_pools() { + for shard in 0..pool.shards() { + for server in 0..pool.servers(shard) { + let address = pool.address(shard, server); + if let Some(server_info) = server_stats_by_addresses.get(&address.name()) { + let metrics = [ + ("bytes_received", server_info.bytes_received), + ("bytes_sent", server_info.bytes_sent), + ("transaction_count", server_info.transaction_count), + ("query_count", server_info.query_count), + ("error_count", server_info.error_count), + ]; + for (key, value) in metrics { + if let Some(prometheus_metric) = + PrometheusMetric::::from_server_info(address, key, value) + { + lines.push(prometheus_metric.to_string()); + } else { + warn!("Metric {} not implemented for {}", key, address.name()); + } + } + } + } + } + } +} + pub async fn start_metric_server(http_addr: SocketAddr) { let http_service_factory = make_service_fn(|_conn| async { Ok::<_, hyper::Error>(service_fn(prometheus_stats)) }); From 6ba21029bc9836aa94d1262ab1db856989ef275e Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 19 Jan 2023 20:36:48 -0500 Subject: [PATCH 305/551] Refactors is_banned logic and forces health check on unban (#288) * Refactors is_banned logic and forces healthcheck on unban * typo * Make is banned log debug * addressing comments * Comment --- src/admin.rs | 2 +- src/pool.rs | 239 ++++++++++++++++++++++++++------------------ src/query_router.rs | 1 + 3 files changed, 145 insertions(+), 97 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 5879114ac..9d4526ecc 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -286,7 +286,7 @@ where for server in 0..pool.servers(shard) { let address = pool.address(shard, server); let pool_state = pool.pool_state(shard, server); - let banned = pool.is_banned(address, Some(address.role)); + let banned = pool.is_banned(address); res.put(data_row(&vec![ address.name(), // name diff --git a/src/pool.rs b/src/pool.rs index 82720aaa9..0f9215887 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -91,6 +91,9 @@ pub struct PoolSettings { // Health check delay pub healthcheck_delay: u64, + + // Ban time + pub ban_time: i64, } impl Default for PoolSettings { @@ -107,6 +110,7 @@ impl Default for PoolSettings { automatic_sharding_key: None, healthcheck_delay: General::default_healthcheck_delay(), healthcheck_timeout: General::default_healthcheck_timeout(), + ban_time: General::default_ban_time(), } } } @@ -277,6 +281,7 @@ impl ConnectionPool { automatic_sharding_key: pool_config.automatic_sharding_key.clone(), healthcheck_delay: config.general.healthcheck_delay, healthcheck_timeout: config.general.healthcheck_timeout, + ban_time: config.general.ban_time, }, }; @@ -352,9 +357,9 @@ impl ConnectionPool { /// Get a connection from the pool. pub async fn get( &self, - shard: usize, // shard number - role: Option, // primary or replica - process_id: i32, // client id + shard: usize, // shard number + role: Option, // primary or replica + client_process_id: i32, // client id ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { let mut candidates: Vec<&Address> = self.addresses[shard] .iter() @@ -380,14 +385,20 @@ impl ConnectionPool { None => break, }; - if self.is_banned(address, role) { - debug!("Address {:?} is banned", address); - continue; + let mut force_healthcheck = false; + + if self.is_banned(address) { + if self.try_unban(&address).await { + force_healthcheck = true; + } else { + debug!("Address {:?} is banned", address); + continue; + } } // Indicate we're waiting on a server connection from a pool. let now = Instant::now(); - self.stats.client_waiting(process_id); + self.stats.client_waiting(client_process_id); // Check if we can connect let mut conn = match self.databases[address.shard][address.address_index] @@ -397,8 +408,9 @@ impl ConnectionPool { Ok(conn) => conn, Err(err) => { error!("Banning instance {:?}, error: {:?}", address, err); - self.ban(address, process_id); - self.stats.client_checkout_error(process_id, address.id); + self.ban(address, client_process_id); + self.stats + .client_checkout_error(client_process_id, address.id); continue; } }; @@ -407,83 +419,105 @@ impl ConnectionPool { let server = &mut *conn; // Will return error if timestamp is greater than current system time, which it should never be set to - let require_healthcheck = server.last_activity().elapsed().unwrap().as_millis() - > self.settings.healthcheck_delay as u128; + let require_healthcheck = force_healthcheck + || server.last_activity().elapsed().unwrap().as_millis() + > self.settings.healthcheck_delay as u128; // Do not issue a health check unless it's been a little while // since we last checked the server is ok. // Health checks are pretty expensive. if !require_healthcheck { + self.stats.checkout_time( + now.elapsed().as_micros(), + client_process_id, + server.server_id(), + ); self.stats - .checkout_time(now.elapsed().as_micros(), process_id, server.server_id()); - self.stats.server_active(process_id, server.server_id()); + .server_active(client_process_id, server.server_id()); return Ok((conn, address.clone())); } - debug!("Running health check on server {:?}", address); - - self.stats.server_tested(server.server_id()); - - match tokio::time::timeout( - tokio::time::Duration::from_millis(self.settings.healthcheck_timeout), - server.query(";"), // Cheap query as it skips the query planner - ) - .await + if self + .run_health_check(address, server, now, client_process_id) + .await { - // Check if health check succeeded. - Ok(res) => match res { - Ok(_) => { - self.stats.checkout_time( - now.elapsed().as_micros(), - process_id, - conn.server_id(), - ); - self.stats.server_active(process_id, conn.server_id()); - return Ok((conn, address.clone())); - } - - // Health check failed. - Err(err) => { - error!( - "Banning instance {:?} because of failed health check, {:?}", - address, err - ); + return Ok((conn, address.clone())); + } else { + continue; + } + } - // Don't leave a bad connection in the pool. - server.mark_bad(); + Err(Error::AllServersDown) + } - self.ban(address, process_id); - continue; - } - }, + async fn run_health_check( + &self, + address: &Address, + server: &mut Server, + start: Instant, + client_process_id: i32, + ) -> bool { + debug!("Running health check on server {:?}", address); + + self.stats.server_tested(server.server_id()); + + match tokio::time::timeout( + tokio::time::Duration::from_millis(self.settings.healthcheck_timeout), + server.query(";"), // Cheap query as it skips the query planner + ) + .await + { + // Check if health check succeeded. + Ok(res) => match res { + Ok(_) => { + self.stats.checkout_time( + start.elapsed().as_micros(), + client_process_id, + server.server_id(), + ); + self.stats + .server_active(client_process_id, server.server_id()); + return true; + } - // Health check timed out. + // Health check failed. Err(err) => { error!( - "Banning instance {:?} because of health check timeout, {:?}", + "Banning instance {:?} because of failed health check, {:?}", address, err ); - // Don't leave a bad connection in the pool. - server.mark_bad(); - - self.ban(address, process_id); - continue; } + }, + + // Health check timed out. + Err(err) => { + error!( + "Banning instance {:?} because of health check timeout, {:?}", + address, err + ); } } - Err(Error::AllServersDown) + // Don't leave a bad connection in the pool. + server.mark_bad(); + + self.ban(&address, client_process_id); + return false; } /// Ban an address (i.e. replica). It no longer will serve /// traffic for any new transactions. Existing transactions on that replica /// will finish successfully or error out to the clients. pub fn ban(&self, address: &Address, client_id: i32) { - error!("Banning {:?}", address); - self.stats.client_ban_error(client_id, address.id); + // Primary can never be banned + if address.role == Role::Primary { + return; + } let now = chrono::offset::Utc::now().naive_utc(); let mut guard = self.banlist.write(); + error!("Banning {:?}", address); + self.stats.client_ban_error(client_id, address.id); guard[address.shard].insert(address.clone(), now); } @@ -494,55 +528,68 @@ impl ConnectionPool { guard[address.shard].remove(address); } - /// Check if a replica can serve traffic. If all replicas are banned, - /// we unban all of them. Better to try then not to. - pub fn is_banned(&self, address: &Address, role: Option) -> bool { - let replicas_available = match role { - Some(Role::Replica) => self.addresses[address.shard] - .iter() - .filter(|addr| addr.role == Role::Replica) - .count(), - None => self.addresses[address.shard].len(), - Some(Role::Primary) => return false, // Primary cannot be banned. - }; + /// Check if address is banned + /// true if banned, false otherwise + pub fn is_banned(&self, address: &Address) -> bool { + let guard = self.banlist.read(); - debug!("Available targets for {:?}: {}", role, replicas_available); + match guard[address.shard].get(address) { + Some(_) => true, + None => { + debug!("{:?} is ok", address); + false + } + } + } - let guard = self.banlist.read(); + /// Determines trying to unban this server was successful + pub async fn try_unban(&self, address: &Address) -> bool { + // If somehow primary ends up being banned we should return true here + if address.role == Role::Primary { + return true; + } + + // Check if all replicas are banned, in that case unban all of them + let replicas_available = self.addresses[address.shard] + .iter() + .filter(|addr| addr.role == Role::Replica) + .count(); - // Everything is banned = nothing is banned. - if guard[address.shard].len() == replicas_available { - drop(guard); - let mut guard = self.banlist.write(); - guard[address.shard].clear(); - drop(guard); + debug!("Available targets: {}", replicas_available); + + let read_guard = self.banlist.read(); + let all_replicas_banned = read_guard[address.shard].len() == replicas_available; + drop(read_guard); + + if all_replicas_banned { + let mut write_guard = self.banlist.write(); warn!("Unbanning all replicas."); - return false; + write_guard[address.shard].clear(); + + return true; } - // I expect this to miss 99.9999% of the time. - match guard[address.shard].get(address) { + // Check if ban time is expired + let read_guard = self.banlist.read(); + let exceeded_ban_time = match read_guard[address.shard].get(address) { Some(timestamp) => { let now = chrono::offset::Utc::now().naive_utc(); - let config = get_config(); - - // Ban expired. - if now.timestamp() - timestamp.timestamp() > config.general.ban_time { - drop(guard); - warn!("Unbanning {:?}", address); - let mut guard = self.banlist.write(); - guard[address.shard].remove(address); - false - } else { - debug!("{:?} is banned", address); - true - } - } - - None => { - debug!("{:?} is ok", address); - false + now.timestamp() - timestamp.timestamp() > self.settings.ban_time } + None => return true, + }; + drop(read_guard); + + if exceeded_ban_time { + warn!("Unbanning {:?}", address); + let mut write_guard = self.banlist.write(); + write_guard[address.shard].remove(address); + drop(write_guard); + + true + } else { + debug!("{:?} is banned", address); + false } } diff --git a/src/query_router.rs b/src/query_router.rs index 9f9dcd76e..28d899dce 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -774,6 +774,7 @@ mod test { automatic_sharding_key: Some(String::from("id")), healthcheck_delay: PoolSettings::default().healthcheck_delay, healthcheck_timeout: PoolSettings::default().healthcheck_timeout, + ban_time: PoolSettings::default().ban_time, }; let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); From c340882bf4ecf47219270f56a85050a2dcfc2acd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Jan 2023 21:14:48 -0800 Subject: [PATCH 306/551] chore(deps): bump toml from 0.5.10 to 0.5.11 (#290) Bumps [toml](https://github.com/toml-rs/toml) from 0.5.10 to 0.5.11. - [Release notes](https://github.com/toml-rs/toml/releases) - [Commits](https://github.com/toml-rs/toml/compare/toml-v0.5.10...toml-v0.5.11) --- updated-dependencies: - dependency-name: toml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08a9a2159..f1f2f5d23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1099,9 +1099,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] From d3aede767749d34d36b5c9ffa696394dec16c7e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 08:01:16 -0800 Subject: [PATCH 307/551] chore(deps): bump async-trait from 0.1.61 to 0.1.63 (#291) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.61 to 0.1.63. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/compare/0.1.61...0.1.63) --- updated-dependencies: - dependency-name: async-trait dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1f2f5d23..f33b264f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-trait" -version = "0.1.61" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" +checksum = "eff18d764974428cf3a9328e23fc5c986f5fbed46e6cd4cdf42544df5d297ec1" dependencies = [ "proc-macro2", "quote", From 187a7b2b5f51a08fd03cd31e58097f86ad14ffd3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Jan 2023 11:48:21 -0800 Subject: [PATCH 308/551] chore(deps): bump toml from 0.5.11 to 0.6.0 (#292) Bumps [toml](https://github.com/toml-rs/toml) from 0.5.11 to 0.6.0. - [Release notes](https://github.com/toml-rs/toml/releases) - [Commits](https://github.com/toml-rs/toml/compare/toml-v0.5.11...toml-v0.6.0) --- updated-dependencies: - dependency-name: toml dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] --- Cargo.lock | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 +- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f33b264f8..276436e9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -586,6 +586,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "nom8" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +dependencies = [ + "memchr", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -890,6 +899,9 @@ name = "serde" version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" @@ -902,6 +914,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_spanned" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c68e921cef53841b8925c2abadd27c9b891d9613bdc43d6b823062866df38e8" +dependencies = [ + "serde", +] + [[package]] name = "sha-1" version = "0.10.1" @@ -1099,13 +1120,38 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.11" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729bfd096e40da9c001f778f5cdecbd2957929a24e10e5883d9392220a751581" +dependencies = [ + "indexmap", + "nom8", + "serde", + "serde_spanned", + "toml_datetime", +] + [[package]] name = "tower-service" version = "0.3.2" diff --git a/Cargo.toml b/Cargo.toml index 3123a6aa2..5dde943f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ async-trait = "0.1" rand = "0.8" chrono = "0.4" sha-1 = "0.10" -toml = "0.5" +toml = "0.6" serde = "1" serde_derive = "1" regex = "1" From b39e18d90d5064461c48f2cdc656a4352a81bdb9 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 28 Jan 2023 09:17:49 -0800 Subject: [PATCH 309/551] Fix formatting (#299) --- src/main.rs | 267 ++++++++++++++++++++++++++-------------------------- 1 file changed, 134 insertions(+), 133 deletions(-) diff --git a/src/main.rs b/src/main.rs index aac51d67b..823186939 100644 --- a/src/main.rs +++ b/src/main.rs @@ -123,103 +123,105 @@ fn main() -> Result<(), Box> { runtime.block_on(async move { - if let Some(true) = config.general.enable_prometheus_exporter { + if let Some(true) = config.general.enable_prometheus_exporter { let http_addr_str = format!( - "{}:{}", - config.general.host, config.general.prometheus_exporter_port + "{}:{}", + config.general.host, config.general.prometheus_exporter_port ); + let http_addr = match SocketAddr::from_str(&http_addr_str) { - Ok(addr) => addr, - Err(err) => { + Ok(addr) => addr, + Err(err) => { error!("Invalid http address: {}", err); std::process::exit(exitcode::CONFIG); - } + } }; + tokio::task::spawn(async move { - start_metric_server(http_addr).await; + start_metric_server(http_addr).await; }); - } + } - let addr = format!("{}:{}", config.general.host, config.general.port); + let addr = format!("{}:{}", config.general.host, config.general.port); - let listener = match TcpListener::bind(&addr).await { + let listener = match TcpListener::bind(&addr).await { Ok(sock) => sock, Err(err) => { - error!("Listener socket error: {:?}", err); - std::process::exit(exitcode::CONFIG); + error!("Listener socket error: {:?}", err); + std::process::exit(exitcode::CONFIG); } - }; + }; - info!("Running on {}", addr); + info!("Running on {}", addr); - config.show(); + config.show(); - // Tracks which client is connected to which server for query cancellation. - let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); + // Tracks which client is connected to which server for query cancellation. + let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); - // Statistics reporting. - let (stats_tx, stats_rx) = mpsc::channel(100_000); - REPORTER.store(Arc::new(Reporter::new(stats_tx.clone()))); + // Statistics reporting. + let (stats_tx, stats_rx) = mpsc::channel(100_000); + REPORTER.store(Arc::new(Reporter::new(stats_tx.clone()))); - // Connection pool that allows to query all shards and replicas. - match ConnectionPool::from_config(client_server_map.clone()).await { + // Connection pool that allows to query all shards and replicas. + match ConnectionPool::from_config(client_server_map.clone()).await { Ok(_) => (), Err(err) => { - error!("Pool error: {:?}", err); - std::process::exit(exitcode::CONFIG); + error!("Pool error: {:?}", err); + std::process::exit(exitcode::CONFIG); } - }; + }; - tokio::task::spawn(async move { + tokio::task::spawn(async move { let mut stats_collector = Collector::new(stats_rx, stats_tx.clone()); stats_collector.collect().await; - }); + }); + + info!("Config autoreloader: {}", config.general.autoreload); - info!("Config autoreloader: {}", config.general.autoreload); + let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); + let autoreload_client_server_map = client_server_map.clone(); - let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); - let autoreload_client_server_map = client_server_map.clone(); - tokio::task::spawn(async move { + tokio::task::spawn(async move { loop { - autoreload_interval.tick().await; - if config.general.autoreload { + autoreload_interval.tick().await; + if config.general.autoreload { info!("Automatically reloading config"); if let Ok(changed) = reload_config(autoreload_client_server_map.clone()).await { - if changed { + if changed { get_config().show() - } + } }; - } + } } - }); - - let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); - let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); - let mut sighup_signal = unix_signal(SignalKind::hangup()).unwrap(); - let (shutdown_tx, _) = broadcast::channel::<()>(1); - let (drain_tx, mut drain_rx) = mpsc::channel::(2048); - let (exit_tx, mut exit_rx) = mpsc::channel::<()>(1); + }); - info!("Waiting for clients"); + let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); + let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); + let mut sighup_signal = unix_signal(SignalKind::hangup()).unwrap(); + let (shutdown_tx, _) = broadcast::channel::<()>(1); + let (drain_tx, mut drain_rx) = mpsc::channel::(2048); + let (exit_tx, mut exit_rx) = mpsc::channel::<()>(1); + let mut admin_only = false; + let mut total_clients = 0; - let mut admin_only = false; - let mut total_clients = 0; + info!("Waiting for clients"); - loop { + loop { tokio::select! { - // Reload config: - // kill -SIGHUP $(pgrep pgcat) - _ = sighup_signal.recv() => { + // Reload config: + // kill -SIGHUP $(pgrep pgcat) + _ = sighup_signal.recv() => { info!("Reloading config"); _ = reload_config(client_server_map.clone()).await; get_config().show(); - }, + }, - // Initiate graceful shutdown sequence on sig int - _ = interrupt_signal.recv() => { + // Initiate graceful shutdown sequence on sig int + _ = interrupt_signal.recv() => { info!("Got SIGINT, waiting for client connection drain now"); admin_only = true; @@ -229,101 +231,100 @@ fn main() -> Result<(), Box> { let _ = drain_tx.send(0).await; tokio::task::spawn(async move { - let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(config.general.shutdown_timeout)); + let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(config.general.shutdown_timeout)); - // First tick fires immediately. - interval.tick().await; + // First tick fires immediately. + interval.tick().await; - // Second one in the interval time. - interval.tick().await; + // Second one in the interval time. + interval.tick().await; - // We're done waiting. - error!("Graceful shutdown timed out. {} active clients being closed", total_clients); + // We're done waiting. + error!("Graceful shutdown timed out. {} active clients being closed", total_clients); - let _ = exit_tx.send(()).await; + let _ = exit_tx.send(()).await; }); - }, + }, - _ = term_signal.recv() => { + _ = term_signal.recv() => { info!("Got SIGTERM, closing with {} clients active", total_clients); break; - }, - - new_client = listener.accept() => { - let (socket, addr) = match new_client { - Ok((socket, addr)) => (socket, addr), - Err(err) => { - error!("{:?}", err); - continue; - } - }; - - let shutdown_rx = shutdown_tx.subscribe(); - let drain_tx = drain_tx.clone(); - let client_server_map = client_server_map.clone(); - - let tls_certificate = config.general.tls_certificate.clone(); - - tokio::task::spawn(async move { - let start = chrono::offset::Utc::now().naive_utc(); - - match client::client_entrypoint( - socket, - client_server_map, - shutdown_rx, - drain_tx, - admin_only, - tls_certificate.clone(), - config.general.log_client_connections, - ) - .await - { - Ok(()) => { - - let duration = chrono::offset::Utc::now().naive_utc() - start; - - if config.general.log_client_disconnections { - info!( - "Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); - } else { - debug!( - "Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); - } + }, + + new_client = listener.accept() => { + let (socket, addr) = match new_client { + Ok((socket, addr)) => (socket, addr), + Err(err) => { + error!("{:?}", err); + continue; + } + }; + + let shutdown_rx = shutdown_tx.subscribe(); + let drain_tx = drain_tx.clone(); + let client_server_map = client_server_map.clone(); + + let tls_certificate = config.general.tls_certificate.clone(); + + tokio::task::spawn(async move { + let start = chrono::offset::Utc::now().naive_utc(); + + match client::client_entrypoint( + socket, + client_server_map, + shutdown_rx, + drain_tx, + admin_only, + tls_certificate.clone(), + config.general.log_client_connections, + ) + .await + { + Ok(()) => { + let duration = chrono::offset::Utc::now().naive_utc() - start; + + if config.general.log_client_disconnections { + info!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + } else { + debug!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); } + } - Err(err) => { - match err { - errors::Error::ClientBadStartup => debug!("Client disconnected with error {:?}", err), - _ => warn!("Client disconnected with error {:?}", err), - } - + Err(err) => { + match err { + errors::Error::ClientBadStartup => debug!("Client disconnected with error {:?}", err), + _ => warn!("Client disconnected with error {:?}", err), } - }; - }); - } - _ = exit_rx.recv() => { - break; - } + } + }; + }); + } - client_ping = drain_rx.recv() => { - let client_ping = client_ping.unwrap(); - total_clients += client_ping; + _ = exit_rx.recv() => { + break; + } - if total_clients == 0 && admin_only { - let _ = exit_tx.send(()).await; - } - } + client_ping = drain_rx.recv() => { + let client_ping = client_ping.unwrap(); + total_clients += client_ping; + + if total_clients == 0 && admin_only { + let _ = exit_tx.send(()).await; + } } - } + } + } - info!("Shutting down..."); + info!("Shutting down..."); }); Ok(()) } From fba7f8fe34f7fe46a789aaa8842d03c9b587d664 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 28 Jan 2023 15:36:35 -0800 Subject: [PATCH 310/551] Startup improvements & PAUSE/RESUME (#300) * Dont require servers to be online to start pooler * PAUSE/RESUME * fix * Refresh pool * Fixes * lint --- .circleci/run_tests.sh | 11 ++++ Cargo.lock | 49 +++++++++++++++ Cargo.toml | 1 + src/admin.rs | 119 ++++++++++++++++++++++++++++++++--- src/client.rs | 66 +++++++++++++++----- src/pool.rs | 137 +++++++++++++++++++++++++++++------------ src/server.rs | 1 + 7 files changed, 322 insertions(+), 62 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index c27bb3b99..00d492734 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -56,6 +56,17 @@ psql -U sharding_user -h 127.0.0.1 -p 6432 -c 'COPY (SELECT * FROM pgbench_accou sleep 1 killall psql -s SIGINT +# Pause/resume test. +# Running benches before, during, and after pause/resume. +pgbench -U sharding_user -t 500 -c 2 -h 127.0.0.1 -p 6432 --protocol extended & +BENCH_ONE=$! +PGPASSWORD=admin_pass psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'PAUSE sharded_db,sharding_user' +pgbench -U sharding_user -h 127.0.0.1 -p 6432 -t 500 -c 2 --protocol extended & +BENCH_TWO=$! +PGPASSWORD=admin_pass psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RESUME sharded_db,sharding_user' +wait ${BENCH_ONE} +wait ${BENCH_TWO} + # Reload pool (closing unused server connections) PGPASSWORD=admin_pass psql -U admin_user -h 127.0.0.1 -p 6432 -d pgbouncer -c 'RELOAD' diff --git a/Cargo.lock b/Cargo.lock index 276436e9b..2b8ce70cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -258,6 +258,21 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" +[[package]] +name = "futures" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.25" @@ -265,6 +280,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -273,6 +289,34 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +[[package]] +name = "futures-executor" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" + +[[package]] +name = "futures-macro" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.25" @@ -293,7 +337,11 @@ checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -665,6 +713,7 @@ dependencies = [ "chrono", "env_logger", "exitcode", + "futures", "hmac", "hyper", "jemallocator", diff --git a/Cargo.toml b/Cargo.toml index 5dde943f0..344b1c03b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ rustls-pemfile = "1" hyper = { version = "0.14", features = ["full"] } phf = { version = "0.11.1", features = ["macros"] } exitcode = "1.1.2" +futures = "0.3" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/src/admin.rs b/src/admin.rs index 9d4526ecc..71d3e486d 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -7,7 +7,7 @@ use tokio::time::Instant; use crate::config::{get_config, reload_config, VERSION}; use crate::errors::Error; use crate::messages::*; -use crate::pool::get_all_pools; +use crate::pool::{get_all_pools, get_pool}; use crate::stats::{ get_address_stats, get_client_stats, get_pool_stats, get_server_stats, ClientState, ServerState, }; @@ -44,15 +44,13 @@ where } let len = query.get_i32() as usize; - let query = String::from_utf8_lossy(&query[..len - 5]) - .to_string() - .to_ascii_uppercase(); + let query = String::from_utf8_lossy(&query[..len - 5]).to_string(); trace!("Admin query: {}", query); let query_parts: Vec<&str> = query.trim_end_matches(';').split_whitespace().collect(); - match query_parts[0] { + match query_parts[0].to_ascii_uppercase().as_str() { "RELOAD" => { trace!("RELOAD"); reload(stream, client_server_map).await @@ -61,7 +59,15 @@ where trace!("SET"); ignore_set(stream).await } - "SHOW" => match query_parts[1] { + "PAUSE" => { + trace!("PAUSE"); + pause(stream, query_parts[1]).await + } + "RESUME" => { + trace!("RESUME"); + resume(stream, query_parts[1]).await + } + "SHOW" => match query_parts[1].to_ascii_uppercase().as_str() { "CONFIG" => { trace!("SHOW CONFIG"); show_config(stream).await @@ -287,6 +293,7 @@ where let address = pool.address(shard, server); let pool_state = pool.pool_state(shard, server); let banned = pool.is_banned(address); + let paused = pool.paused(); res.put(data_row(&vec![ address.name(), // name @@ -300,7 +307,11 @@ where pool_config.pool_mode.to_string(), // pool_mode pool_config.user.pool_size.to_string(), // max_connections pool_state.connections.to_string(), // current_connections - "0".to_string(), // paused + match paused { + // paused + true => "1".to_string(), + false => "0".to_string(), + }, match banned { // disabled true => "1".to_string(), @@ -561,3 +572,97 @@ where write_all_half(stream, &res).await } + +/// Pause a pool. It won't pass any more queries to the backends. +async fn pause(stream: &mut T, query: &str) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let parts: Vec<&str> = query.split(",").map(|part| part.trim()).collect(); + + if parts.len() != 2 { + error_response( + stream, + "PAUSE requires a database and a user, e.g. PAUSE my_db,my_user", + ) + .await + } else { + let database = parts[0]; + let user = parts[1]; + + match get_pool(database, user) { + Some(pool) => { + pool.pause(); + + let mut res = BytesMut::new(); + + res.put(command_complete(&format!("PAUSE {},{}", database, user))); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await + } + + None => { + error_response( + stream, + &format!( + "No pool configured for database: {}, user: {}", + database, user + ), + ) + .await + } + } + } +} + +/// Resume a pool. Queries are allowed again. +async fn resume(stream: &mut T, query: &str) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let parts: Vec<&str> = query.split(",").map(|part| part.trim()).collect(); + + if parts.len() != 2 { + error_response( + stream, + "RESUME requires a database and a user, e.g. RESUME my_db,my_user", + ) + .await + } else { + let database = parts[0]; + let user = parts[1]; + + match get_pool(database, user) { + Some(pool) => { + pool.resume(); + + let mut res = BytesMut::new(); + + res.put(command_complete(&format!("RESUME {},{}", database, user))); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await + } + + None => { + error_response( + stream, + &format!( + "No pool configured for database: {}, user: {}", + database, user + ), + ) + .await + } + } + } +} diff --git a/src/client.rs b/src/client.rs index 15fe21d91..ed2044d4d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -476,7 +476,7 @@ where } // Authenticate normal user. else { - let pool = match get_pool(pool_name, username) { + let mut pool = match get_pool(pool_name, username) { Some(pool) => pool, None => { error_response( @@ -504,6 +504,25 @@ where let transaction_mode = pool.settings.pool_mode == PoolMode::Transaction; + // If the pool hasn't been validated yet, + // connect to the servers and figure out what's what. + if !pool.validated() { + match pool.validate().await { + Ok(_) => (), + Err(err) => { + error_response( + &mut write, + &format!( + "Pool down for database: {:?}, user: {:?}", + pool_name, username + ), + ) + .await?; + return Err(Error::ClientError(format!("Pool down: {:?}", err))); + } + } + } + (transaction_mode, pool.server_info()) }; @@ -674,22 +693,16 @@ where // Get a pool instance referenced by the most up-to-date // pointer. This ensures we always read the latest config // when starting a query. - let pool = match get_pool(&self.pool_name, &self.username) { - Some(pool) => pool, - None => { - error_response( - &mut self.write, - &format!( - "No pool configured for database: {:?}, user: {:?}", - self.pool_name, self.username - ), - ) - .await?; + let mut pool = self.get_pool().await?; + + // Check if the pool is paused and wait until it's resumed. + if pool.wait_paused().await { + // Refresh pool information, something might have changed. + pool = self.get_pool().await?; + } - return Err(Error::ClientError(format!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", self.pool_name, self.username, self.application_name))); - } - }; query_router.update_pool_settings(pool.settings.clone()); + let current_shard = query_router.shard(); // Handle all custom protocol commands, if any. @@ -1012,6 +1025,29 @@ where } } + /// Retrieve connection pool, if it exists. + /// Return an error to the client otherwise. + async fn get_pool(&mut self) -> Result { + match get_pool(&self.pool_name, &self.username) { + Some(pool) => Ok(pool), + None => { + error_response( + &mut self.write, + &format!( + "No pool configured for database: {}, user: {}", + self.pool_name, self.username + ), + ) + .await?; + + Err(Error::ClientError(format!( + "Invalid pool name {{ username: {}, pool_name: {}, application_name: {} }}", + self.pool_name, self.username, self.application_name + ))) + } + } + } + /// Release the server from the client: it can't cancel its queries anymore. pub fn release(&self) { let mut guard = self.client_server_map.lock(); diff --git a/src/pool.rs b/src/pool.rs index 0f9215887..702b61781 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,7 +1,7 @@ use arc_swap::ArcSwap; use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection}; -use bytes::BytesMut; +use bytes::{BufMut, BytesMut}; use chrono::naive::NaiveDateTime; use log::{debug, error, info, warn}; use once_cell::sync::Lazy; @@ -9,8 +9,12 @@ use parking_lot::{Mutex, RwLock}; use rand::seq::SliceRandom; use rand::thread_rng; use std::collections::{HashMap, HashSet}; -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; use std::time::Instant; +use tokio::sync::Notify; use crate::config::{get_config, Address, General, LoadBalancingMode, PoolMode, Role, User}; use crate::errors::Error; @@ -56,6 +60,12 @@ impl PoolIdentifier { } } +impl From<&Address> for PoolIdentifier { + fn from(address: &Address) -> PoolIdentifier { + PoolIdentifier::new(&address.database, &address.username) + } +} + /// Pool settings. #[derive(Clone, Debug)] pub struct PoolSettings { @@ -136,10 +146,18 @@ pub struct ConnectionPool { /// The server information (K messages) have to be passed to the /// clients on startup. We pre-connect to all shards and replicas /// on pool creation and save the K messages here. - server_info: BytesMut, + server_info: Arc>, /// Pool configuration. pub settings: PoolSettings, + + /// If not validated, we need to double check the pool is available before allowing a client + /// to use it. + validated: Arc, + + /// If the pool has been paused or not. + paused: Arc, + paused_waiter: Arc, } impl ConnectionPool { @@ -257,12 +275,12 @@ impl ConnectionPool { assert_eq!(shards.len(), addresses.len()); - let mut pool = ConnectionPool { + let pool = ConnectionPool { databases: shards, addresses, banlist: Arc::new(RwLock::new(banlist)), stats: get_reporter(), - server_info: BytesMut::new(), + server_info: Arc::new(RwLock::new(BytesMut::new())), settings: PoolSettings { pool_mode: pool_config.pool_mode, load_balancing_mode: pool_config.load_balancing_mode, @@ -283,17 +301,18 @@ impl ConnectionPool { healthcheck_timeout: config.general.healthcheck_timeout, ban_time: config.general.ban_time, }, + validated: Arc::new(AtomicBool::new(false)), + paused: Arc::new(AtomicBool::new(false)), + paused_waiter: Arc::new(Notify::new()), }; // Connect to the servers to make sure pool configuration is valid // before setting it globally. - match pool.validate().await { - Ok(_) => (), - Err(err) => { - error!("Could not validate connection pool: {:?}", err); - return Err(err); - } - }; + // Do this async and somewhere else, we don't have to wait here. + let mut validate_pool = pool.clone(); + tokio::task::spawn(async move { + let _ = validate_pool.validate().await; + }); // There is one pool per database/user pair. new_pools.insert(PoolIdentifier::new(pool_name, &user.username), pool); @@ -311,49 +330,87 @@ impl ConnectionPool { /// when they connect. /// This also warms up the pool for clients that connect when /// the pooler starts up. - async fn validate(&mut self) -> Result<(), Error> { - let mut server_infos = Vec::new(); + pub async fn validate(&mut self) -> Result<(), Error> { + let mut futures = Vec::new(); + let validated = Arc::clone(&self.validated); + for shard in 0..self.shards() { for server in 0..self.servers(shard) { - let connection = match self.databases[shard][server].get().await { - Ok(conn) => conn, - Err(err) => { - error!("Shard {} down or misconfigured: {:?}", shard, err); - continue; - } - }; + let databases = self.databases.clone(); + let validated = Arc::clone(&validated); + let pool_server_info = Arc::clone(&self.server_info); + + let task = tokio::task::spawn(async move { + let connection = match databases[shard][server].get().await { + Ok(conn) => conn, + Err(err) => { + error!("Shard {} down or misconfigured: {:?}", shard, err); + return; + } + }; - let proxy = connection; - let server = &*proxy; - let server_info = server.server_info(); + let proxy = connection; + let server = &*proxy; + let server_info = server.server_info(); - if !server_infos.is_empty() { - // Compare against the last server checked. - if server_info != server_infos[server_infos.len() - 1] { - warn!( - "{:?} has different server configuration than the last server", - proxy.address() - ); - } - } + let mut guard = pool_server_info.write(); + guard.clear(); + guard.put(server_info.clone()); + validated.store(true, Ordering::Relaxed); + }); - server_infos.push(server_info); + futures.push(task); } } + futures::future::join_all(futures).await; + // TODO: compare server information to make sure // all shards are running identical configurations. - if server_infos.is_empty() { + if self.server_info.read().is_empty() { + error!("Could not validate connection pool"); return Err(Error::AllServersDown); } - // We're assuming all servers are identical. - // TODO: not true. - self.server_info = server_infos[0].clone(); - Ok(()) } + /// The pool can be used by clients. + /// + /// If not, we need to validate it first by connecting to servers. + /// Call `validate()` to do so. + pub fn validated(&self) -> bool { + self.validated.load(Ordering::Relaxed) + } + + /// Pause the pool, allowing no more queries and make clients wait. + pub fn pause(&self) { + self.paused.store(true, Ordering::Relaxed); + } + + /// Resume the pool, allowing queries and resuming any pending queries. + pub fn resume(&self) { + self.paused.store(false, Ordering::Relaxed); + self.paused_waiter.notify_waiters(); + } + + /// Check if the pool is paused. + pub fn paused(&self) -> bool { + self.paused.load(Ordering::Relaxed) + } + + /// Check if the pool is paused and wait until it's resumed. + pub async fn wait_paused(&self) -> bool { + let waiter = self.paused_waiter.notified(); + let paused = self.paused.load(Ordering::Relaxed); + + if paused { + waiter.await; + } + + paused + } + /// Get a connection from the pool. pub async fn get( &self, @@ -624,7 +681,7 @@ impl ConnectionPool { } pub fn server_info(&self) -> BytesMut { - self.server_info.clone() + self.server_info.read().clone() } fn busy_connection_count(&self, address: &Address) -> u32 { diff --git a/src/server.rs b/src/server.rs index f2a6d387d..96004854d 100644 --- a/src/server.rs +++ b/src/server.rs @@ -546,6 +546,7 @@ impl Server { /// If the server is still inside a transaction. /// If the client disconnects while the server is in a transaction, we will clean it up. pub fn in_transaction(&self) -> bool { + debug!("Server in transaction: {}", self.in_transaction); self.in_transaction } From 4fab21a456f817d9ec97642839d584934e5da5c4 Mon Sep 17 00:00:00 2001 From: Kurtsley <73447098+Kurtsley@users.noreply.github.com> Date: Sat, 28 Jan 2023 17:51:05 -0600 Subject: [PATCH 311/551] Add initial Windows support, ref #298 (#301) --- src/main.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index 823186939..cd408c491 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,11 +48,11 @@ use log::{debug, error, info, warn}; use parking_lot::Mutex; use pgcat::format_duration; use tokio::net::TcpListener; -use tokio::{ - runtime::Builder, - signal::unix::{signal as unix_signal, SignalKind}, - sync::mpsc, -}; +#[cfg(not(windows))] +use tokio::signal::unix::{signal as unix_signal, SignalKind}; +#[cfg(windows)] +use tokio::signal::windows as win_signal; +use tokio::{runtime::Builder, sync::mpsc}; use std::collections::HashMap; use std::net::SocketAddr; @@ -197,8 +197,18 @@ fn main() -> Result<(), Box> { } }); + #[cfg(windows)] + let mut term_signal = win_signal::ctrl_close().unwrap(); + #[cfg(windows)] + let mut interrupt_signal = win_signal::ctrl_c().unwrap(); + #[cfg(windows)] + let mut sighup_signal = win_signal::ctrl_shutdown().unwrap(); + + #[cfg(not(windows))] let mut term_signal = unix_signal(SignalKind::terminate()).unwrap(); + #[cfg(not(windows))] let mut interrupt_signal = unix_signal(SignalKind::interrupt()).unwrap(); + #[cfg(not(windows))] let mut sighup_signal = unix_signal(SignalKind::hangup()).unwrap(); let (shutdown_tx, _) = broadcast::channel::<()>(1); let (drain_tx, mut drain_rx) = mpsc::channel::(2048); From 488e0bcbcedc4695d65bff8de261d2eb7a5c8a0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 08:59:16 -0800 Subject: [PATCH 312/551] chore(deps): bump tokio from 1.24.2 to 1.25.0 (#304) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.24.2 to 1.25.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/commits/tokio-1.25.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b8ce70cd..ec8be4c0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1113,9 +1113,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.24.2" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", "bytes", From a3deae921db9aded36c01f5e28e5d2e4b4ded602 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 08:59:37 -0800 Subject: [PATCH 313/551] chore(deps): bump toml from 0.6.0 to 0.7.0 (#305) Bumps [toml](https://github.com/toml-rs/toml) from 0.6.0 to 0.7.0. - [Release notes](https://github.com/toml-rs/toml/releases) - [Commits](https://github.com/toml-rs/toml/compare/toml-v0.6.0...toml-v0.7.0) --- updated-dependencies: - dependency-name: toml dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 15 ++++++--------- Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec8be4c0c..09cc7747c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -948,9 +948,6 @@ name = "serde" version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" -dependencies = [ - "serde_derive", -] [[package]] name = "serde_derive" @@ -1169,9 +1166,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb9d890e4dc9298b70f740f615f2e05b9db37dce531f6b24fb77ac993f9f217" +checksum = "2f560bc7fb3eb31f5eee1340c68a2160cad39605b7b9c9ec32045ddbdee13b85" dependencies = [ "serde", "serde_spanned", @@ -1181,18 +1178,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" +checksum = "886f31a9b85b6182cabd4d8b07df3b451afcc216563748201490940d2a28ed36" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729bfd096e40da9c001f778f5cdecbd2957929a24e10e5883d9392220a751581" +checksum = "233d8716cdc5d20ec88a18a839edaf545edc71efa4a5ff700ef4a102c26cd8fa" dependencies = [ "indexmap", "nom8", diff --git a/Cargo.toml b/Cargo.toml index 344b1c03b..06622db2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ async-trait = "0.1" rand = "0.8" chrono = "0.4" sha-1 = "0.10" -toml = "0.6" +toml = "0.7" serde = "1" serde_derive = "1" regex = "1" From 80fd1e907dba40be3b7571bbc7402a44eb09fd8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:35:14 -0800 Subject: [PATCH 314/551] chore(deps): bump bytes from 1.3.0 to 1.4.0 (#310) Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.3.0 to 1.4.0. - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/compare/v1.3.0...v1.4.0) --- updated-dependencies: - dependency-name: bytes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09cc7747c..e7f616154 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,9 +85,9 @@ checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cc" From 42562688a82c96c039873fdc68a7798140322e00 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:14:54 -0800 Subject: [PATCH 315/551] chore(deps): bump hyper from 0.14.23 to 0.14.24 (#311) Bumps [hyper](https://github.com/hyperium/hyper) from 0.14.23 to 0.14.24. - [Release notes](https://github.com/hyperium/hyper/releases) - [Changelog](https://github.com/hyperium/hyper/blob/v0.14.24/CHANGELOG.md) - [Commits](https://github.com/hyperium/hyper/compare/v0.14.23...v0.14.24) --- updated-dependencies: - dependency-name: hyper dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e7f616154..fe0e33e54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -453,9 +453,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" dependencies = [ "bytes", "futures-channel", From 35049d98fd5c46efed28737c3bda8a5a5737e68a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:15:04 -0800 Subject: [PATCH 316/551] chore(deps): bump toml from 0.7.0 to 0.7.1 (#309) Bumps [toml](https://github.com/toml-rs/toml) from 0.7.0 to 0.7.1. - [Release notes](https://github.com/toml-rs/toml/releases) - [Commits](https://github.com/toml-rs/toml/compare/toml-v0.7.0...toml-v0.7.1) --- updated-dependencies: - dependency-name: toml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe0e33e54..1cac425ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -962,9 +962,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c68e921cef53841b8925c2abadd27c9b891d9613bdc43d6b823062866df38e8" +checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" dependencies = [ "serde", ] @@ -1166,9 +1166,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f560bc7fb3eb31f5eee1340c68a2160cad39605b7b9c9ec32045ddbdee13b85" +checksum = "772c1426ab886e7362aedf4abc9c0d1348a979517efedfc25862944d10137af0" dependencies = [ "serde", "serde_spanned", @@ -1178,18 +1178,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "886f31a9b85b6182cabd4d8b07df3b451afcc216563748201490940d2a28ed36" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233d8716cdc5d20ec88a18a839edaf545edc71efa4a5ff700ef4a102c26cd8fa" +checksum = "90a238ee2e6ede22fb95350acc78e21dc40da00bb66c0334bde83de4ed89424e" dependencies = [ "indexmap", "nom8", From f15c4db829055f30306205ec581167660422cd1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:15:12 -0800 Subject: [PATCH 317/551] chore(deps): bump futures from 0.3.25 to 0.3.26 (#307) Bumps [futures](https://github.com/rust-lang/futures-rs) from 0.3.25 to 0.3.26. - [Release notes](https://github.com/rust-lang/futures-rs/releases) - [Changelog](https://github.com/rust-lang/futures-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/futures-rs/compare/0.3.25...0.3.26) --- updated-dependencies: - dependency-name: futures dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1cac425ff..0a7a1255a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -260,9 +260,9 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" [[package]] name = "futures" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" dependencies = [ "futures-channel", "futures-core", @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", "futures-sink", @@ -285,15 +285,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-executor" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" dependencies = [ "futures-core", "futures-task", @@ -302,15 +302,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" [[package]] name = "futures-macro" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", @@ -319,21 +319,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures-channel", "futures-core", From 4438012dd78c678fda355f29d09ae88f0a988b56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Feb 2023 17:15:19 -0800 Subject: [PATCH 318/551] chore(deps): bump async-trait from 0.1.63 to 0.1.64 (#308) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.63 to 0.1.64. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/compare/0.1.63...0.1.64) --- updated-dependencies: - dependency-name: async-trait dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a7a1255a..0715cd834 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-trait" -version = "0.1.63" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff18d764974428cf3a9328e23fc5c986f5fbed46e6cd4cdf42544df5d297ec1" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2", "quote", From 33d86c6027b4f554e79ed52b71f9ff39aad5ca6e Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Tue, 7 Feb 2023 15:16:28 -0500 Subject: [PATCH 319/551] Fix logging mistakes (#313) Mistakenly logging username as poolname and poolname as username --- src/client.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/client.rs b/src/client.rs index ed2044d4d..11f4cc67e 100644 --- a/src/client.rs +++ b/src/client.rs @@ -432,7 +432,7 @@ where let code = match read.read_u8().await { Ok(p) => p, - Err(_) => return Err(Error::SocketError(format!("Error reading password code from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))), + Err(_) => return Err(Error::SocketError(format!("Error reading password code from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))), }; // PasswordMessage @@ -445,14 +445,14 @@ where let len = match read.read_i32().await { Ok(len) => len, - Err(_) => return Err(Error::SocketError(format!("Error reading password message length from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))), + Err(_) => return Err(Error::SocketError(format!("Error reading password message length from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))), }; let mut password_response = vec![0u8; (len - 4) as usize]; match read.read_exact(&mut password_response).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError(format!("Error reading password message from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))), + Err(_) => return Err(Error::SocketError(format!("Error reading password message from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))), }; // Authenticate admin user. @@ -466,10 +466,10 @@ where ); if password_hash != password_response { - warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name); + warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name); wrong_password(&mut write, username).await?; - return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))); + return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))); } (false, generate_server_info_for_admin()) @@ -488,7 +488,7 @@ where ) .await?; - return Err(Error::ClientError(format!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))); + return Err(Error::ClientError(format!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))); } }; @@ -496,10 +496,10 @@ where let password_hash = md5_hash_password(username, &pool.settings.user.password, &salt); if password_hash != password_response { - warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name); + warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name); wrong_password(&mut write, username).await?; - return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", pool_name, username, application_name))); + return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))); } let transaction_mode = pool.settings.pool_mode == PoolMode::Transaction; From ab702a9306b383f16119cbfecdbaa7edfa8ec57b Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 8 Feb 2023 11:35:38 -0600 Subject: [PATCH 320/551] Introduce tcp_keepalives to PgCat (#315) We have encountered a case where PgCat pools were stuck following a database incident. Our best understanding at this point is that the PgCat -> Postgres connections died silently and because Tokio defaults to disabling keepalives, connections in the pool were marked as busy forever. Only when we deployed PgCat did we see recovery. This PR introduces tcp_keepalives to PgCat. This sets the defaults to be keepalives_idle: 5 # seconds keepalives_interval: 5 # seconds keepalives_count: 5 # a count These settings can detect the death of an idle connection within 30 seconds of its death. Please note that the connection can remain idle forever (from an application perspective) as long as the keepalive packets are flowing so disconnection will only occur if the other end is not acknowledging keepalive packets (keepalive packet acks are handled by the OS, the application does not need to do anything). I plan to add tcp_user_timeout in a follow-up PR. --- .circleci/config.yml | 2 +- Cargo.lock | 1 + Cargo.toml | 1 + src/config.rs | 25 ++++++++++++++ src/messages.rs | 24 ++++++++++++++ src/server.rs | 2 ++ tests/docker/Dockerfile | 2 +- tests/ruby/helpers/pgcat_helper.rb | 12 +++---- tests/ruby/helpers/pgcat_process.rb | 5 ++- tests/ruby/load_balancing_spec.rb | 2 +- tests/ruby/misc_spec.rb | 51 +++++++++++++++++++++++++++-- 11 files changed, 114 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 98aaea714..ab0d77393 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,7 +54,7 @@ jobs: command: "cargo fmt --check" - run: name: "Install dependencies" - command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip lcov llvm-11 && sudo apt-get upgrade curl" + command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip lcov llvm-11 iproute2 && sudo apt-get upgrade curl" - run: name: "Install rust tools" command: "cargo install cargo-binutils rustfilt && rustup component add llvm-tools-preview" diff --git a/Cargo.lock b/Cargo.lock index 0715cd834..6afbe9e10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -730,6 +730,7 @@ dependencies = [ "serde_derive", "sha-1", "sha2", + "socket2", "sqlparser", "stringprep", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 06622db2b..3cf0e7aa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ hyper = { version = "0.14", features = ["full"] } phf = { version = "0.11.1", features = ["macros"] } exitcode = "1.1.2" futures = "0.3" +socket2 = { version = "0.4.7", features = ["all"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/src/config.rs b/src/config.rs index 219f0debc..694342303 100644 --- a/src/config.rs +++ b/src/config.rs @@ -160,6 +160,13 @@ pub struct General { #[serde(default = "General::default_idle_timeout")] pub idle_timeout: u64, + #[serde(default = "General::default_tcp_keepalives_idle")] + pub tcp_keepalives_idle: u64, + #[serde(default = "General::default_tcp_keepalives_count")] + pub tcp_keepalives_count: u32, + #[serde(default = "General::default_tcp_keepalives_interval")] + pub tcp_keepalives_interval: u64, + #[serde(default)] // False pub log_client_connections: bool, @@ -203,6 +210,21 @@ impl General { 1000 } + // These keepalive defaults should detect a dead connection within 30 seconds. + // Tokio defaults to disabling keepalives which keeps dead connections around indefinitely. + // This can lead to permenant server pool exhaustion + pub fn default_tcp_keepalives_idle() -> u64 { + 5 // 5 seconds + } + + pub fn default_tcp_keepalives_count() -> u32 { + 5 // 5 time + } + + pub fn default_tcp_keepalives_interval() -> u64 { + 5 // 5 seconds + } + pub fn default_idle_timeout() -> u64 { 60000 // 10 minutes } @@ -242,6 +264,9 @@ impl Default for General { healthcheck_delay: Self::default_healthcheck_delay(), ban_time: Self::default_ban_time(), worker_threads: Self::default_worker_threads(), + tcp_keepalives_idle: Self::default_tcp_keepalives_idle(), + tcp_keepalives_count: Self::default_tcp_keepalives_count(), + tcp_keepalives_interval: Self::default_tcp_keepalives_interval(), log_client_connections: false, log_client_disconnections: false, autoreload: false, diff --git a/src/messages.rs b/src/messages.rs index e7c36747a..3fc84b5a9 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -1,14 +1,18 @@ /// Helper functions to send one-off protocol messages /// and handle TcpStream (TCP socket). use bytes::{Buf, BufMut, BytesMut}; +use log::error; use md5::{Digest, Md5}; +use socket2::{SockRef, TcpKeepalive}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; +use crate::config::get_config; use crate::errors::Error; use std::collections::HashMap; use std::io::{BufRead, Cursor}; use std::mem; +use std::time::Duration; /// Postgres data type mappings /// used in RowDescription ('T') message. @@ -550,6 +554,26 @@ pub fn server_parameter_message(key: &str, value: &str) -> BytesMut { server_info } +pub fn configure_socket(stream: &TcpStream) { + let sock_ref = SockRef::from(stream); + let conf = get_config(); + + match sock_ref.set_keepalive(true) { + Ok(_) => { + match sock_ref.set_tcp_keepalive( + &TcpKeepalive::new() + .with_interval(Duration::from_secs(conf.general.tcp_keepalives_interval)) + .with_retries(conf.general.tcp_keepalives_count) + .with_time(Duration::from_secs(conf.general.tcp_keepalives_idle)), + ) { + Ok(_) => (), + Err(err) => error!("Could not configure socket: {}", err), + } + } + Err(err) => error!("Could not configure socket: {}", err), + } +} + pub trait BytesMutReader { fn read_string(&mut self) -> Result; } diff --git a/src/server.rs b/src/server.rs index 96004854d..1d9bcd14f 100644 --- a/src/server.rs +++ b/src/server.rs @@ -92,6 +92,7 @@ impl Server { ))); } }; + configure_socket(&stream); trace!("Sending StartupMessage"); @@ -368,6 +369,7 @@ impl Server { return Err(Error::SocketError(format!("Error reading cancel message"))); } }; + configure_socket(&stream); debug!("Sending CancelRequest"); diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 1f11efb3e..1ff55567a 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -1,5 +1,5 @@ FROM rust:bullseye -RUN apt-get update && apt-get install llvm-11 psmisc postgresql-contrib postgresql-client ruby ruby-dev libpq-dev python3 python3-pip lcov sudo curl -y +RUN apt-get update && apt-get install llvm-11 psmisc postgresql-contrib postgresql-client ruby ruby-dev libpq-dev python3 python3-pip lcov curl sudo iproute2 -y RUN cargo install cargo-binutils rustfilt RUN rustup component add llvm-tools-preview diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index ffa60953b..544c8278b 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -5,7 +5,7 @@ module Helpers module Pgcat - def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random") + def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random", log_level="info") user = { "password" => "sharding_user", "pool_size" => pool_size, @@ -13,7 +13,7 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mod "username" => "sharding_user" } - pgcat = PgcatProcess.new("info") + pgcat = PgcatProcess.new(log_level) primary0 = PgInstance.new(5432, user["username"], user["password"], "shard0") primary1 = PgInstance.new(7432, user["username"], user["password"], "shard1") primary2 = PgInstance.new(8432, user["username"], user["password"], "shard2") @@ -47,7 +47,7 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mod end end - def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random") + def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random", log_level="trace") user = { "password" => "sharding_user", "pool_size" => pool_size, @@ -55,7 +55,7 @@ def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction", lb "username" => "sharding_user" } - pgcat = PgcatProcess.new("trace") + pgcat = PgcatProcess.new(log_level) pgcat_cfg = pgcat.current_config primary = PgInstance.new(5432, user["username"], user["password"], "shard0") @@ -92,7 +92,7 @@ def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction", lb end end - def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random") + def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random", log_level="info") user = { "password" => "sharding_user", "pool_size" => pool_size, @@ -100,7 +100,7 @@ def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mo "username" => "sharding_user" } - pgcat = PgcatProcess.new("info") + pgcat = PgcatProcess.new(log_level) pgcat_cfg = pgcat.current_config primary = PgInstance.new(5432, user["username"], user["password"], "shard0") diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index b6e798aed..00d72bd56 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -8,7 +8,7 @@ class PgcatProcess attr_reader :pid def self.finalize(pid, log_filename, config_filename) - `kill #{pid}` + `kill #{pid}` if pid File.delete(config_filename) if File.exist?(config_filename) File.delete(log_filename) if File.exist?(log_filename) end @@ -75,8 +75,11 @@ def wait_until_ready end def stop + return unless @pid + `kill #{@pid}` sleep 0.1 + @pid = nil end def shutdown diff --git a/tests/ruby/load_balancing_spec.rb b/tests/ruby/load_balancing_spec.rb index fccf0a859..5e088d15f 100644 --- a/tests/ruby/load_balancing_spec.rb +++ b/tests/ruby/load_balancing_spec.rb @@ -88,7 +88,7 @@ end context "under heterogeneous load" do - it "balances query volume between all instances based on how busy they are" do + xit "balances query volume between all instances based on how busy they are" do slow_query_count = 2 threads = Array.new(slow_query_count) do Thread.new do diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index 1f5bf4217..be7af42b5 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -8,6 +8,51 @@ processes.pgcat.shutdown end + describe "TCP Keepalives" do + # Ideally, we should block TCP traffic to the database using + # iptables to mimic passive (connection is dropped without a RST packet) + # but we cannot do this in CircleCI because iptables requires NET_ADMIN + # capability that we cannot enable in CircleCI + # Toxiproxy won't work either because it does not block keepalives + # so our best bet is to query the OS keepalive params set on the socket + + context "default settings" do + it "applies default keepalive settings" do + # We query ss command to verify that we have correct keepalive values set + # we can only verify the keepalives_idle parameter but that's good enough + # example output + #Recv-Q Send-Q Local Address:Port Peer Address:Port Process + #0 0 127.0.0.1:60526 127.0.0.1:18432 timer:(keepalive,1min59sec,0) + #0 0 127.0.0.1:60664 127.0.0.1:19432 timer:(keepalive,4.123ms,0) + + port_search_criteria = processes.all_databases.map { |d| "dport = :#{d.port}"}.join(" or ") + results = `ss -t4 state established -o -at '( #{port_search_criteria} )'`.lines + results.shift + results.each { |line| expect(line).to match(/timer:\(keepalive,.*ms,0\)/) } + end + end + + context "changed settings" do + it "applies keepalive settings from config" do + new_configs = processes.pgcat.current_config + + new_configs["general"]["tcp_keepalives_idle"] = 120 + new_configs["general"]["tcp_keepalives_count"] = 1 + new_configs["general"]["tcp_keepalives_interval"] = 1 + processes.pgcat.update_config(new_configs) + # We need to kill the old process that was using the default configs + processes.pgcat.stop + processes.pgcat.start + processes.pgcat.wait_until_ready + + port_search_criteria = processes.all_databases.map { |d| "dport = :#{d.port}"}.join(" or ") + results = `ss -t4 state established -o -at '( #{port_search_criteria} )'`.lines + results.shift + results.each { |line| expect(line).to include("timer:(keepalive,1min") } + end + end + end + describe "Extended Protocol handling" do it "does not send packets that client does not expect during extended protocol sequence" do new_configs = processes.pgcat.current_config @@ -189,7 +234,7 @@ expect(processes.primary.count_query("DISCARD ALL")).to eq(10) end end - + context "transaction mode with transactions" do let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5, "transaction") } it "Does not clear set statement state when declared in a transaction" do @@ -200,7 +245,7 @@ conn.async_exec("SET statement_timeout to 1000") conn.async_exec("COMMIT") conn.close - end + end expect(processes.primary.count_query("DISCARD ALL")).to eq(0) 10.times do @@ -210,7 +255,7 @@ conn.async_exec("SET LOCAL statement_timeout to 1000") conn.async_exec("COMMIT") conn.close - end + end expect(processes.primary.count_query("DISCARD ALL")).to eq(0) end end From 629c6c78f6eec508b393cb15047e6743be7a507c Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 13 Feb 2023 15:29:08 -0600 Subject: [PATCH 321/551] Fix code coverage + less flakiness (#318) Code coverage logic was missing coverage from rust tests. This is now fixed. Also, we weren't reaping spawned PgCat processes correctly which left zombie processes. --- .circleci/generate_coverage.sh | 10 +++++++++- tests/ruby/helpers/pgcat_process.rb | 10 +++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.circleci/generate_coverage.sh b/.circleci/generate_coverage.sh index 59fe44aa8..0a21e1c66 100755 --- a/.circleci/generate_coverage.sh +++ b/.circleci/generate_coverage.sh @@ -1,7 +1,15 @@ #!/bin/bash +# inspired by https://doc.rust-lang.org/rustc/instrument-coverage.html#tips-for-listing-the-binaries-automatically +TEST_OBJECTS=$( \ + for file in $(cargo test --no-run 2>&1 | grep "target/debug/deps/pgcat-[[:alnum:]]\+" -o); \ + do \ + printf "%s %s " --object $file; \ + done \ +) + rust-profdata merge -sparse pgcat-*.profraw -o pgcat.profdata -rust-cov export -ignore-filename-regex="rustc|registry" -Xdemangler=rustfilt -instr-profile=pgcat.profdata --object ./target/debug/pgcat --format lcov > ./lcov.info +bash -c "rust-cov export -ignore-filename-regex='rustc|registry' -Xdemangler=rustfilt -instr-profile=pgcat.profdata $TEST_OBJECTS --object ./target/debug/pgcat --format lcov > ./lcov.info" genhtml lcov.info --output-directory /tmp/cov --prefix $(pwd) diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index 00d72bd56..63b510452 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -8,7 +8,11 @@ class PgcatProcess attr_reader :pid def self.finalize(pid, log_filename, config_filename) - `kill #{pid}` if pid + if pid + Process.kill("TERM", pid) + Process.wait(pid) + end + File.delete(config_filename) if File.exist?(config_filename) File.delete(log_filename) if File.exist?(log_filename) end @@ -77,8 +81,8 @@ def wait_until_ready def stop return unless @pid - `kill #{@pid}` - sleep 0.1 + Process.kill("TERM", @pid) + Process.wait(@pid) @pid = nil end From bae1f010edacc11b767cc54edc8a71e07286afa8 Mon Sep 17 00:00:00 2001 From: Tommy Chen Date: Wed, 15 Feb 2023 00:30:47 +0800 Subject: [PATCH 322/551] Support EC and PKCS8 private keys (#316) * Support EC and PKCS8 private keys * Use iter instead of infinite loop in `load_keys` fn --- src/tls.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/tls.rs b/src/tls.rs index dbbfcdec0..fbfbae759 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -1,6 +1,7 @@ // Stream wrapper. -use rustls_pemfile::{certs, rsa_private_keys}; +use rustls_pemfile::{certs, read_one, Item}; +use std::iter; use std::path::Path; use std::sync::Arc; use tokio_rustls::rustls::{self, Certificate, PrivateKey}; @@ -17,9 +18,17 @@ pub fn load_certs(path: &Path) -> std::io::Result> { } pub fn load_keys(path: &Path) -> std::io::Result> { - rsa_private_keys(&mut std::io::BufReader::new(std::fs::File::open(path)?)) - .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid key")) - .map(|mut keys| keys.drain(..).map(PrivateKey).collect()) + let mut rd = std::io::BufReader::new(std::fs::File::open(path)?); + + iter::from_fn(|| read_one(&mut rd).transpose()) + .filter_map(|item| match item { + Err(err) => Some(Err(err)), + Ok(Item::RSAKey(key)) => Some(Ok(PrivateKey(key))), + Ok(Item::ECKey(key)) => Some(Ok(PrivateKey(key))), + Ok(Item::PKCS8Key(key)) => Some(Ok(PrivateKey(key))), + _ => None, + }) + .collect() } pub struct Tls { From 51d983e4ab2ef3b98671e1e7299fdf399614dd7b Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 14 Feb 2023 16:20:11 -0600 Subject: [PATCH 323/551] Fix deprecation warnings (#319) warning: use of deprecated function `base64::decode`: Use Engine::decode --- src/scram.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/scram.rs b/src/scram.rs index c3f920da9..3e5d84705 100644 --- a/src/scram.rs +++ b/src/scram.rs @@ -2,6 +2,7 @@ // https://github.com/sfackler/rust-postgres/ // SASL implementation. +use base64::{engine::general_purpose, Engine as _}; use bytes::BytesMut; use hmac::{Hmac, Mac}; use rand::{self, Rng}; @@ -81,7 +82,7 @@ impl ScramSha256 { return Err(Error::ProtocolSyncError(format!("SCRAM"))); } - let salt = match base64::decode(&server_message.salt) { + let salt = match general_purpose::STANDARD.decode(&server_message.salt) { Ok(salt) => salt, Err(_) => return Err(Error::ProtocolSyncError(format!("SCRAM"))), }; @@ -111,7 +112,7 @@ impl ScramSha256 { let mut cbind_input = vec![]; cbind_input.extend("n,,".as_bytes()); - let cbind_input = base64::encode(&cbind_input); + let cbind_input = general_purpose::STANDARD.encode(&cbind_input); self.message.clear(); @@ -149,7 +150,11 @@ impl ScramSha256 { *proof ^= signature; } - match write!(&mut self.message, ",p={}", base64::encode(&*client_proof)) { + match write!( + &mut self.message, + ",p={}", + general_purpose::STANDARD.encode(&*client_proof) + ) { Ok(_) => (), Err(_) => return Err(Error::ServerError), }; @@ -161,7 +166,7 @@ impl ScramSha256 { pub fn finish(&mut self, message: &BytesMut) -> Result<(), Error> { let final_message = FinalMessage::parse(message)?; - let verifier = match base64::decode(&final_message.value) { + let verifier = match general_purpose::STANDARD.decode(&final_message.value) { Ok(verifier) => verifier, Err(_) => return Err(Error::ProtocolSyncError(format!("SCRAM"))), }; From f4a5f0ac0ea39eb2be16c0b8397469e720b4c121 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 21:43:47 -0800 Subject: [PATCH 324/551] chore(deps): bump once_cell from 1.17.0 to 1.17.1 (#320) Bumps [once_cell](https://github.com/matklad/once_cell) from 1.17.0 to 1.17.1. - [Release notes](https://github.com/matklad/once_cell/releases) - [Changelog](https://github.com/matklad/once_cell/blob/master/CHANGELOG.md) - [Commits](https://github.com/matklad/once_cell/compare/v1.17.0...v1.17.1) --- updated-dependencies: - dependency-name: once_cell dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6afbe9e10..7b18b2c1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -674,9 +674,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "parking_lot" From 3c5a4f283516cb2e628fbd54360803a442b7b453 Mon Sep 17 00:00:00 2001 From: John Meagher Date: Wed, 15 Feb 2023 15:19:16 -0600 Subject: [PATCH 325/551] Allow shard setting with comments (#293) What Allows shard selection by the client to come in via comments like /* shard_id: 1 */ select * from foo; Why We're using a setup in Ruby that makes it tough or impossible to inject commands on the connection to set the shard before it gets to the "real" SQL being run. Instead we have an updated PG adapter that allows injection of comments before each executed SQL statement. We need this support in pgcat in order to keep some complex shard picking logic in Ruby code while using pgcat for connection management. Local Testing Run postgres and pgcat with the default options. Run psql < tests/sharding/query_routing_setup.sql to setup the database for the tests and run ./tests/pgbench/external_shard_test.sh as often as needed to exercise the shard setting comment test. --- pgcat.toml | 6 +++ src/config.rs | 23 +++++++++ src/pool.rs | 22 +++++++++ src/query_router.rs | 116 +++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 159 insertions(+), 8 deletions(-) diff --git a/pgcat.toml b/pgcat.toml index 0187c16b5..26f9d7ddc 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -85,6 +85,12 @@ query_parser_enabled = true # queries. The primary can always be explicitly selected with our custom protocol. primary_reads_enabled = true +# Allow sharding commands to be passed as statement comments instead of +# separate commands. If these are unset this functionality is disabled. +# sharding_key_regex = '/\* sharding_key: (\d+) \*/' +# shard_id_regex = '/\* shard_id: (\d+) \*/' +# regex_search_limit = 1000 # only look at the first 1000 characters of SQL statements + # So what if you wanted to implement a different hashing function, # or you've already built one and you want this pooler to use it? # diff --git a/src/config.rs b/src/config.rs index 694342303..392acfbbe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,6 +2,7 @@ use arc_swap::ArcSwap; use log::{error, info}; use once_cell::sync::Lazy; +use regex::Regex; use serde_derive::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::hash::Hash; @@ -342,8 +343,15 @@ pub struct Pool { #[serde(default = "Pool::default_automatic_sharding_key")] pub automatic_sharding_key: Option, + pub sharding_key_regex: Option, + pub shard_id_regex: Option, + pub regex_search_limit: Option, + pub shards: BTreeMap, pub users: BTreeMap, + // Note, don't put simple fields below these configs. There's a compatability issue with TOML that makes it + // incompatible to have simple fields in TOML after complex objects. See + // https://users.rust-lang.org/t/why-toml-to-string-get-error-valueaftertable/85903 } impl Pool { @@ -387,6 +395,18 @@ impl Pool { shard.validate()?; } + for (option, name) in [ + (&self.shard_id_regex, "shard_id_regex"), + (&self.sharding_key_regex, "sharding_key_regex"), + ] { + if let Some(regex) = option { + if let Err(parse_err) = Regex::new(regex.as_str()) { + error!("{} is not a valid Regex: {}", name, parse_err); + return Err(Error::BadConfig); + } + } + } + Ok(()) } } @@ -405,6 +425,9 @@ impl Default for Pool { automatic_sharding_key: None, connect_timeout: None, idle_timeout: None, + sharding_key_regex: None, + shard_id_regex: None, + regex_search_limit: Some(1000), } } } diff --git a/src/pool.rs b/src/pool.rs index 702b61781..cbe5b5007 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -8,6 +8,7 @@ use once_cell::sync::Lazy; use parking_lot::{Mutex, RwLock}; use rand::seq::SliceRandom; use rand::thread_rng; +use regex::Regex; use std::collections::{HashMap, HashSet}; use std::sync::{ atomic::{AtomicBool, Ordering}, @@ -104,6 +105,15 @@ pub struct PoolSettings { // Ban time pub ban_time: i64, + + // Regex for searching for the sharding key in SQL statements + pub sharding_key_regex: Option, + + // Regex for searching for the shard id in SQL statements + pub shard_id_regex: Option, + + // Limit how much of each query is searched for a potential shard regex match + pub regex_search_limit: usize, } impl Default for PoolSettings { @@ -121,6 +131,9 @@ impl Default for PoolSettings { healthcheck_delay: General::default_healthcheck_delay(), healthcheck_timeout: General::default_healthcheck_timeout(), ban_time: General::default_ban_time(), + sharding_key_regex: None, + shard_id_regex: None, + regex_search_limit: 1000, } } } @@ -300,6 +313,15 @@ impl ConnectionPool { healthcheck_delay: config.general.healthcheck_delay, healthcheck_timeout: config.general.healthcheck_timeout, ban_time: config.general.ban_time, + sharding_key_regex: pool_config + .sharding_key_regex + .clone() + .map(|regex| Regex::new(regex.as_str()).unwrap()), + shard_id_regex: pool_config + .shard_id_regex + .clone() + .map(|regex| Regex::new(regex.as_str()).unwrap()), + regex_search_limit: pool_config.regex_search_limit.unwrap_or(1000), }, validated: Arc::new(AtomicBool::new(false)), paused: Arc::new(AtomicBool::new(false)), diff --git a/src/query_router.rs b/src/query_router.rs index 28d899dce..bf07db7fe 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -14,6 +14,7 @@ use crate::messages::BytesMutReader; use crate::pool::PoolSettings; use crate::sharding::Sharder; +use std::cmp; use std::collections::BTreeSet; use std::io::Cursor; @@ -114,7 +115,52 @@ impl QueryRouter { let code = message_cursor.get_u8() as char; - // Only simple protocol supported for commands. + // Check for any sharding regex matches in any queries + match code as char { + // For Parse and Query messages peek to see if they specify a shard_id as a comment early in the statement + 'P' | 'Q' => { + if self.pool_settings.shard_id_regex.is_some() + || self.pool_settings.sharding_key_regex.is_some() + { + // Check only the first block of bytes configured by the pool settings + let len = message_cursor.get_i32() as usize; + let seg = cmp::min(len - 5, self.pool_settings.regex_search_limit); + let initial_segment = String::from_utf8_lossy(&message_buffer[0..seg]); + + // Check for a shard_id included in the query + if let Some(shard_id_regex) = &self.pool_settings.shard_id_regex { + let shard_id = shard_id_regex.captures(&initial_segment).and_then(|cap| { + cap.get(1).and_then(|id| id.as_str().parse::().ok()) + }); + if let Some(shard_id) = shard_id { + debug!("Setting shard to {:?}", shard_id); + self.set_shard(shard_id); + // Skip other command processing since a sharding command was found + return None; + } + } + + // Check for a sharding_key included in the query + if let Some(sharding_key_regex) = &self.pool_settings.sharding_key_regex { + let sharding_key = + sharding_key_regex + .captures(&initial_segment) + .and_then(|cap| { + cap.get(1).and_then(|id| id.as_str().parse::().ok()) + }); + if let Some(sharding_key) = sharding_key { + debug!("Setting sharding_key to {:?}", sharding_key); + self.set_sharding_key(sharding_key); + // Skip other command processing since a sharding command was found + return None; + } + } + } + } + _ => {} + } + + // Only simple protocol supported for commands processed below if code != 'Q' { return None; } @@ -192,13 +238,11 @@ impl QueryRouter { match command { Command::SetShardingKey => { - let sharder = Sharder::new( - self.pool_settings.shards, - self.pool_settings.sharding_function, - ); - let shard = sharder.shard(value.parse::().unwrap()); - self.active_shard = Some(shard); - value = shard.to_string(); + // TODO: some error handling here + value = self + .set_sharding_key(value.parse::().unwrap()) + .unwrap() + .to_string(); } Command::SetShard => { @@ -465,6 +509,16 @@ impl QueryRouter { } } + fn set_sharding_key(&mut self, sharding_key: i64) -> Option { + let sharder = Sharder::new( + self.pool_settings.shards, + self.pool_settings.sharding_function, + ); + let shard = sharder.shard(sharding_key); + self.set_shard(shard); + self.active_shard + } + /// Get the current desired server role we should be talking to. pub fn role(&self) -> Option { self.active_role @@ -775,6 +829,9 @@ mod test { healthcheck_delay: PoolSettings::default().healthcheck_delay, healthcheck_timeout: PoolSettings::default().healthcheck_timeout, ban_time: PoolSettings::default().ban_time, + sharding_key_regex: None, + shard_id_regex: None, + regex_search_limit: 1000, }; let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); @@ -820,4 +877,47 @@ mod test { ))); assert_eq!(qr.role(), Role::Primary); } + + #[test] + fn test_regex_shard_parsing() { + QueryRouter::setup(); + + let pool_settings = PoolSettings { + pool_mode: PoolMode::Transaction, + load_balancing_mode: crate::config::LoadBalancingMode::Random, + shards: 5, + user: crate::config::User::default(), + default_role: Some(Role::Replica), + query_parser_enabled: true, + primary_reads_enabled: false, + sharding_function: ShardingFunction::PgBigintHash, + automatic_sharding_key: Some(String::from("id")), + healthcheck_delay: PoolSettings::default().healthcheck_delay, + healthcheck_timeout: PoolSettings::default().healthcheck_timeout, + ban_time: PoolSettings::default().ban_time, + sharding_key_regex: Some(Regex::new(r"/\* sharding_key: (\d+) \*/").unwrap()), + shard_id_regex: Some(Regex::new(r"/\* shard_id: (\d+) \*/").unwrap()), + regex_search_limit: 1000, + }; + let mut qr = QueryRouter::new(); + qr.update_pool_settings(pool_settings.clone()); + + // Shard should start out unset + assert_eq!(qr.active_shard, None); + + // Make sure setting it works + let q1 = simple_query("/* shard_id: 1 */ select 1 from foo;"); + assert!(qr.try_execute_command(&q1) == None); + assert_eq!(qr.active_shard, Some(1)); + + // And make sure changing it works + let q2 = simple_query("/* shard_id: 0 */ select 1 from foo;"); + assert!(qr.try_execute_command(&q2) == None); + assert_eq!(qr.active_shard, Some(0)); + + // Validate setting by shard with expected shard copied from sharding.rs tests + let q2 = simple_query("/* sharding_key: 6 */ select 1 from foo;"); + assert!(qr.try_execute_command(&q2) == None); + assert_eq!(qr.active_shard, Some(2)); + } } From da4adf70a4f478d117634dc4a61e287fcb959760 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 16 Feb 2023 17:51:38 -0500 Subject: [PATCH 326/551] Log worker thread count at startup (#322) --- src/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.rs b/src/config.rs index 392acfbbe..f911d415a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -621,6 +621,7 @@ impl Config { /// Print current configuration. pub fn show(&self) { info!("Ban time: {}s", self.general.ban_time); + info!("Worker threads: {}", self.general.worker_threads); info!( "Healthcheck timeout: {}ms", self.general.healthcheck_timeout From 0bd2677dc36935372a4323c966ccefcea73db6aa Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 16 Feb 2023 18:34:35 -0600 Subject: [PATCH 327/551] Bake toxiproxy into CI image (#323) Instead of downloading Toxiproxy everytime we run CI, we bake it into the CI docker image --- Dockerfile.ci | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.ci b/Dockerfile.ci index 8f8ab9459..d8febbb22 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -6,3 +6,5 @@ RUN cargo install cargo-binutils rustfilt && \ rustup component add llvm-tools-preview RUN pip3 install psycopg2 && \ sudo gem install bundler +RUN wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ + sudo dpkg -i toxiproxy-2.4.0.deb From a2af27c843449c207c67436848261a7c712a7691 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 16 Feb 2023 20:31:53 -0600 Subject: [PATCH 328/551] Build Dockerfile.ci using Github workflows (#325) We have to build and push the docker image used in CI manually. This PR builds that image automatically and pushes it to Github docker repository. Will start using that image in a follow PR --- .github/workflows/publish-ci-docker-image.yml | 20 +++++++++++++++++++ Dockerfile.ci | 16 ++++++++------- 2 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/publish-ci-docker-image.yml diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml new file mode 100644 index 000000000..3cae8e7e2 --- /dev/null +++ b/.github/workflows/publish-ci-docker-image.yml @@ -0,0 +1,20 @@ +name: publish-ci-docker-image +on: + push: + branches: [ main ] +jobs: + publish-ci-docker-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build CI Docker image + run: | + docker build . -f Dockerfile.ci --tag ghcr.io/levkk/pgcat-ci:latest + docker run ghcr.io/levkk/pgcat-ci:latest + docker push ghcr.io/levkk/pgcat-ci:latest diff --git a/Dockerfile.ci b/Dockerfile.ci index d8febbb22..71d3d738c 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,10 +1,12 @@ FROM cimg/rust:1.62.0 RUN sudo apt-get update && \ - sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip lcov llvm-11 && \ - sudo apt-get upgrade curl -RUN cargo install cargo-binutils rustfilt && \ - rustup component add llvm-tools-preview -RUN pip3 install psycopg2 && \ - sudo gem install bundler -RUN wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ + sudo apt-get install -y \ + postgresql-contrib-12 postgresql-client-12 libpq-dev \ + ruby ruby-dev python3 python3-pip \ + lcov llvm-11 iproute2 && \ + sudo apt-get upgrade curl && \ + cargo install cargo-binutils rustfilt && \ + rustup component add llvm-tools-preview && \ + pip3 install psycopg2 && sudo gem install bundler && \ + wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ sudo dpkg -i toxiproxy-2.4.0.deb From 8cd189b49494d51ed9716f173ad4b7f443f02254 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 16 Feb 2023 21:35:15 -0600 Subject: [PATCH 329/551] Move toxiproxy.deb to /tmp (#326) I am seeing Directory (/home/circleci/project) you are trying to checkout to is not empty and not a git repository error after I started using the new Dockerfile.ci image. My best guess is that this failure is because we download toxiproxy.deb file into the home directory which blocks git checkout. This PR moves toxiproxy to /tmp/ to avoid this --- Dockerfile.ci | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 71d3d738c..4ebeefdd5 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -8,5 +8,5 @@ RUN sudo apt-get update && \ cargo install cargo-binutils rustfilt && \ rustup component add llvm-tools-preview && \ pip3 install psycopg2 && sudo gem install bundler && \ - wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ - sudo dpkg -i toxiproxy-2.4.0.deb + wget -O /tmp/toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ + sudo dpkg -i /tmp/toxiproxy-2.4.0.deb From 809ab2516aafe36cd11be3b4c5f2af15481ac950 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 16 Feb 2023 21:50:03 -0600 Subject: [PATCH 330/551] Add psmisc to CI image (#327) I accidentally removed `psmisc` from the image and now the test builds are failing. I am adding it back in this PR --- Dockerfile.ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 4ebeefdd5..42d213a8c 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,7 +1,7 @@ FROM cimg/rust:1.62.0 RUN sudo apt-get update && \ sudo apt-get install -y \ - postgresql-contrib-12 postgresql-client-12 libpq-dev \ + psmisc postgresql-contrib-12 postgresql-client-12 libpq-dev \ ruby ruby-dev python3 python3-pip \ lcov llvm-11 iproute2 && \ sudo apt-get upgrade curl && \ From 3fd538a562df204dbe9b6d258a0f9ea8afeb74e4 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 16 Feb 2023 23:09:22 -0600 Subject: [PATCH 331/551] More Test coverage + fix some code coverage bugs (#321) Connection to the CI databases is viewed by Postgres as coming from localhost. The pg_hba.conf file generated by the docker image uses trust for these connections, that's why we had no test coverage on SASL and md5 branches. This PR fixes this issue. There was also an issue with under-reporting code coverage. This should be fixed now --- .circleci/config.yml | 28 +++++++++++----------------- .circleci/generate_coverage.sh | 6 +++--- .circleci/pgcat.toml | 4 ++-- .circleci/run_tests.sh | 5 ----- tests/docker/Dockerfile | 3 +++ tests/docker/docker-compose.yml | 11 ++++------- tests/docker/run.sh | 24 ++++++++++++++++++++---- tests/ruby/admin_spec.rb | 2 +- tests/ruby/load_balancing_spec.rb | 2 +- tests/ruby/spec_helper.rb | 2 +- 10 files changed, 46 insertions(+), 41 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab0d77393..f7aa899b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,39 +9,42 @@ jobs: # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - - image: levkk/pgcat-ci:latest + - image: ghcr.io/levkk/pgcat-ci:latest environment: RUST_LOG: info - RUSTFLAGS: "-C instrument-coverage" - LLVM_PROFILE_FILE: "pgcat-%m.profraw" + LLVM_PROFILE_FILE: /tmp/pgcat-%m-%p.profraw + RUSTC_BOOTSTRAP: 1 + CARGO_INCREMENTAL: 0 + RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cinstrument-coverage" + RUSTDOCFLAGS: "-Cpanic=abort" - image: postgres:14 command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements"] environment: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 - image: postgres:14 command: ["postgres", "-p", "7432", "-c", "shared_preload_libraries=pg_stat_statements"] environment: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 - image: postgres:14 command: ["postgres", "-p", "8432", "-c", "shared_preload_libraries=pg_stat_statements"] environment: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 - image: postgres:14 command: ["postgres", "-p", "9432", "-c", "shared_preload_libraries=pg_stat_statements"] environment: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps @@ -52,18 +55,9 @@ jobs: - run: name: "Lint" command: "cargo fmt --check" - - run: - name: "Install dependencies" - command: "sudo apt-get update && sudo apt-get install -y psmisc postgresql-contrib-12 postgresql-client-12 ruby ruby-dev libpq-dev python3 python3-pip lcov llvm-11 iproute2 && sudo apt-get upgrade curl" - - run: - name: "Install rust tools" - command: "cargo install cargo-binutils rustfilt && rustup component add llvm-tools-preview" - - run: - name: "Build" - command: "cargo build" - run: name: "Tests" - command: "cargo test && bash .circleci/run_tests.sh && .circleci/generate_coverage.sh" + command: "cargo clean && cargo build && cargo test && bash .circleci/run_tests.sh && .circleci/generate_coverage.sh" - store_artifacts: path: /tmp/cov destination: coverage-data diff --git a/.circleci/generate_coverage.sh b/.circleci/generate_coverage.sh index 0a21e1c66..f583dd0c3 100755 --- a/.circleci/generate_coverage.sh +++ b/.circleci/generate_coverage.sh @@ -8,8 +8,8 @@ TEST_OBJECTS=$( \ done \ ) -rust-profdata merge -sparse pgcat-*.profraw -o pgcat.profdata +rust-profdata merge -sparse /tmp/pgcat-*.profraw -o /tmp/pgcat.profdata -bash -c "rust-cov export -ignore-filename-regex='rustc|registry' -Xdemangler=rustfilt -instr-profile=pgcat.profdata $TEST_OBJECTS --object ./target/debug/pgcat --format lcov > ./lcov.info" +bash -c "rust-cov export -ignore-filename-regex='rustc|registry' -Xdemangler=rustfilt -instr-profile=/tmp/pgcat.profdata $TEST_OBJECTS --object ./target/debug/pgcat --format lcov > ./lcov.info" -genhtml lcov.info --output-directory /tmp/cov --prefix $(pwd) +genhtml lcov.info -show-details --highlight --ignore-errors source --legend --output-directory /tmp/cov --prefix $(pwd) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 2a959d590..0d47ed722 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -18,10 +18,10 @@ enable_prometheus_exporter = true prometheus_exporter_port = 9930 # How long to wait before aborting a server connection (ms). -connect_timeout = 100 +connect_timeout = 1000 # How much time to give the health check query to return with a result (ms). -healthcheck_timeout = 100 +healthcheck_timeout = 1000 # How long to keep connection available for immediate re-use, without running a healthcheck query on it healthcheck_delay = 30000 diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 00d492734..644f22e83 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -24,10 +24,6 @@ PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard0 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard2 -i -# Install Toxiproxy to simulate a downed/slow database -wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb -sudo dpkg -i toxiproxy-2.4.0.deb - # Start Toxiproxy LOG_LEVEL=error toxiproxy-server & sleep 1 @@ -99,7 +95,6 @@ kill -SIGHUP $(pgrep pgcat) # Reload config again # ActiveRecord tests # cd tests/ruby -sudo gem install bundler sudo bundle install bundle exec ruby tests.rb || exit 1 bundle exec rspec *_spec.rb || exit 1 diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 1ff55567a..99fd694db 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -3,3 +3,6 @@ FROM rust:bullseye RUN apt-get update && apt-get install llvm-11 psmisc postgresql-contrib postgresql-client ruby ruby-dev libpq-dev python3 python3-pip lcov curl sudo iproute2 -y RUN cargo install cargo-binutils rustfilt RUN rustup component add llvm-tools-preview +RUN sudo gem install bundler +RUN wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ + sudo dpkg -i toxiproxy-2.4.0.deb diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index d86e23990..e44dc5292 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -7,7 +7,7 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "5432"] pg2: image: postgres:14 @@ -16,7 +16,7 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "7432"] pg3: image: postgres:14 @@ -25,7 +25,7 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "8432"] pg4: image: postgres:14 @@ -34,14 +34,11 @@ services: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres - POSTGRES_HOST_AUTH_METHOD: scram-sha-256 + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "9432"] main: build: . command: ["bash", "/app/tests/docker/run.sh"] - environment: - RUSTFLAGS: "-C instrument-coverage" - LLVM_PROFILE_FILE: "pgcat-%m.profraw" volumes: - ../../:/app/ - /app/target/ diff --git a/tests/docker/run.sh b/tests/docker/run.sh index ada5d9ee5..5c4a877cb 100644 --- a/tests/docker/run.sh +++ b/tests/docker/run.sh @@ -1,21 +1,37 @@ #!/bin/bash +rm -rf /app/target/ || true rm /app/*.profraw || true rm /app/pgcat.profdata || true rm -rf /app/cov || true -cd /app/ +export LLVM_PROFILE_FILE="/app/pgcat-%m-%p.profraw" +export RUSTC_BOOTSTRAP=1 +export CARGO_INCREMENTAL=0 +export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cinstrument-coverage" +export RUSTDOCFLAGS="-Cpanic=abort" +cd /app/ +cargo clean cargo build cargo test --tests bash .circleci/run_tests.sh -rust-profdata merge -sparse pgcat-*.profraw -o pgcat.profdata +TEST_OBJECTS=$( \ + for file in $(cargo test --no-run 2>&1 | grep "target/debug/deps/pgcat-[[:alnum:]]\+" -o); \ + do \ + printf "%s %s " --object $file; \ + done \ +) + +echo "Generating coverage report" + +rust-profdata merge -sparse /app/pgcat-*.profraw -o /app/pgcat.profdata -rust-cov export -ignore-filename-regex="rustc|registry" -Xdemangler=rustfilt -instr-profile=pgcat.profdata --object ./target/debug/pgcat --format lcov > ./lcov.info +bash -c "rust-cov export -ignore-filename-regex='rustc|registry' -Xdemangler=rustfilt -instr-profile=/app/pgcat.profdata $TEST_OBJECTS --object ./target/debug/pgcat --format lcov > ./lcov.info" -genhtml lcov.info --output-directory cov --prefix $(pwd) +genhtml lcov.info -show-details --highlight --ignore-errors source --legend --output-directory cov --prefix $(pwd) rm /app/*.profraw rm /app/pgcat.profdata diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index 40e7e1c2d..2000e5d96 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -221,7 +221,7 @@ results = admin_conn.async_exec("SHOW POOLS")[0] expect(results["maxwait"]).to eq("1") - expect(results["maxwait_us"].to_i).to be_within(100_000).of(500_000) + expect(results["maxwait_us"].to_i).to be_within(200_000).of(500_000) sleep(4.5) # Allow time for stats to update results = admin_conn.async_exec("SHOW POOLS")[0] diff --git a/tests/ruby/load_balancing_spec.rb b/tests/ruby/load_balancing_spec.rb index 5e088d15f..9c204c392 100644 --- a/tests/ruby/load_balancing_spec.rb +++ b/tests/ruby/load_balancing_spec.rb @@ -46,7 +46,7 @@ end end - expect(failed_count).to eq(2) + expect(failed_count).to be <= 2 processes.all_databases.each do |instance| queries_routed = instance.count_select_1_plus_2 if processes.replicas[0..1].include?(instance) diff --git a/tests/ruby/spec_helper.rb b/tests/ruby/spec_helper.rb index 3050e18aa..d67964017 100644 --- a/tests/ruby/spec_helper.rb +++ b/tests/ruby/spec_helper.rb @@ -4,7 +4,7 @@ require_relative 'helpers/pgcat_helper' QUERY_COUNT = 300 -MARGIN_OF_ERROR = 0.30 +MARGIN_OF_ERROR = 0.35 def with_captured_stdout_stderr sout = STDOUT.clone From 3286d405644962cf07e8d9f2741dad8bc820f056 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 17 Feb 2023 09:18:54 -0600 Subject: [PATCH 332/551] More coverage cleanup (#328) Apply a new style + remove function coverage report --- .circleci/generate_coverage.sh | 2 +- cov-style.css | 158 +++++++++++++++++++++++++++++++++ tests/docker/run.sh | 2 +- 3 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 cov-style.css diff --git a/.circleci/generate_coverage.sh b/.circleci/generate_coverage.sh index f583dd0c3..6244c1926 100755 --- a/.circleci/generate_coverage.sh +++ b/.circleci/generate_coverage.sh @@ -12,4 +12,4 @@ rust-profdata merge -sparse /tmp/pgcat-*.profraw -o /tmp/pgcat.profdata bash -c "rust-cov export -ignore-filename-regex='rustc|registry' -Xdemangler=rustfilt -instr-profile=/tmp/pgcat.profdata $TEST_OBJECTS --object ./target/debug/pgcat --format lcov > ./lcov.info" -genhtml lcov.info -show-details --highlight --ignore-errors source --legend --output-directory /tmp/cov --prefix $(pwd) +genhtml lcov.info --title "PgCat Code Coverage" --css-file ./cov-style.css --no-function-coverage --highlight --ignore-errors source --legend --output-directory /tmp/cov --prefix $(pwd) diff --git a/cov-style.css b/cov-style.css new file mode 100644 index 000000000..681cf9ede --- /dev/null +++ b/cov-style.css @@ -0,0 +1,158 @@ +/* + * Copyright 2021 Collabora, Ltd. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +body { + background-color: #f2f2f2; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, + "Noto Sans", Ubuntu, Cantarell, "Helvetica Neue", sans-serif, + "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", + "Noto Color Emoji"; +} + +.sourceHeading, .source, .coverFn, +.testName, .testPer, .testNum, +.coverLegendCovLo, .headerCovTableEntryLo, .coverPerLo, .coverNumLo, +.coverLegendCovMed, .headerCovTableEntryMed, .coverPerMed, .coverNumMed, +.coverLegendCovHi, .headerCovTableEntryHi, .coverPerHi, .coverNumHi, +.coverFile { + font-family: "Menlo", "DejaVu Sans Mono", "Liberation Mono", + "Consolas", "Ubuntu Mono", "Courier New", "andale mono", + "lucida console", monospace; +} + +pre { + font-size: 0.7875rem; +} + +.headerCovTableEntry, .testPer, .testNum, .testName, +.coverLegendCovLo, .headerCovTableEntryLo, .coverPerLo, .coverNumLo, +.coverLegendCovMed, .headerCovTableEntryMed, .coverPerMed, .coverNumMed, +.coverLegendCovHi, .headerCovTableEntryHi, .coverPerHi, .coverNumHi { + text-align: right; + white-space: nowrap; +} + +.coverPerLo, .coverPerMed, .coverPerHi, .testPer { +/* font-weight: bold;*/ +} + +.coverNumLo, .coverNumMed, .coverNumHi, .testNum { + font-style: italic; + font-size: 90%; + padding-left: 1em; +} + +.title { + font-size: 200%; +} + +.tableHead { + text-align: center; + font-weight: bold; + background-color: #bfbfbf; +} + +.coverFile, .coverBar, .coverFn { + background-color: #d9d9d9; +} + +.headerCovTableHead { + font-weight: bold; + text-align: right; +} + +.headerCovTableEntry { + background-color: #d9d9d9; +} + +.coverFnLo, +.coverLegendCovLo, .headerCovTableEntryLo, .coverPerLo, .coverNumLo { + background-color: #f2dada; +} + +.coverFnHi, +.coverLegendCovMed, .headerCovTableEntryMed, .coverPerMed, .coverNumMed { + background-color: #add9ad; +} + +.coverLegendCovHi, .headerCovTableEntryHi, .coverPerHi, .coverNumHi { + background-color: #59b359; +} + +.coverBarOutline { + border-style: solid; + border-width: 1px; + border-color: black; + padding: 0px; +} + +.coverFnLo, .coverFnHi { + text-align: right; +} + +.lineNum { + background-color: #d9d9d9; +} + +.coverLegendCov, .lineCov, .branchCov { + background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAABCAIAAABsYngUAAADAXpUWHRSYXcgcHJvZmlsZSB0eXBlIGV4aWYAAHjazZVbktwgDEX/WUWWgCSExHIwj6rsIMvPxcY9PY9MzVTyEVMNtCwkoYNwGL9+zvADDxHHkNQ8l5wjnlRS4YqJx+upZ08xnf313O/otTw8FBgzwShbP2/5gJyhz1vetp0KuT4ZKmO/OF6/qNsQ+3ZwO9yOhC4HcRsOdRsS3p7T9f+4thVzcXveQtv6sz5t1dfW0CUxzprJEvrE0SwXzJ1jMuStr0CPvhfqdvTmf7hVGTHxEJKI3leEsn4kFWNCT/CGfUnBXDEuyd4yaHGIhnm58/r581nk4Q59Y32N+p69Qc3xPelwJvRWkTeE8mP8UE76Ig/PSE9uT55z3jN+LZ/pJaibXLjxzdl9znHtrqaMLee9qXuL5wx6x8rWuSqjGX4afSV7tYLmKImGc9RxyA60RoUYGCcl6lRp0jjHRg0hJh4MjszcALcFCB0wCjcgJYBGo8kGzF0cB6DhOAik/IiFTrfldNfI4biTB5wegjHCkr9q4StKc66CIlq55CtXiItXwhHFIkeE6ocaiNDcSdUzwXd7+yyuAoJ6ptmxwRqPZQH4D6WXwyUnaIGiYrwKmKxvA0gRIlAEQwICMZMoZYrGHIwIiXQAqgidJfEBLKTKHUFyEsmAgyqAb6wxOlVZ+RLjIgQIlRzEwAaFCFgpKc6PJccZqiqaVDWrqWvRmiWvCsvZ8rpRq4klU8tm5lasBhdPrp7d3L14LVwEN64W1GPxUkqtcFphuWJ1hUKtBx9ypEOPfNjhRzlq49CkpaYtN2veSqudu3TUcc/duvfS66CBozTS0JGHDR9l1ImjNmWmqTNPmz5LmPVBbWN9175BjTY1PkktRXtQg9TsNkHrOtHFDMQ4EYDbIkASmBez6JQSL3KLWSyMqlBGkLrgdFrEQDANYp30YPdCToPkf8MtAAT/C3JhofsCuffcPqLW6/mhk5PQKsOV1CiovpHgnx3LcCvhwlnz9dF8P4Y/vfju+J8aQpZK+A373P3XzDqcKwAAAAZiS0dEAAAAAAAA+UO7fwAAAAlwSFlzAAAOxAAADsQBlSsOGwAAAAd0SU1FB+UEEQYyDQA04tUAAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAAADklEQVQI12PULVBlwAYAEagAxGHRDdwAAAAASUVORK5CYII='); + background-repeat: repeat-y; + background-position: left top; + background-color: #c6ffb8; +} + +.coverLegendNoCov, .lineNoCov, .branchNoCov, .branchNoExec { + background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAABCAIAAABsYngUAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAAB3RJTUUH5QMUCiMidNgp2gAAABl0RVh0Q29tbWVudABDcmVhdGVkIHdpdGggR0lNUFeBDhcAAAAPSURBVAjXY/wZIcWADQAAIa4BbZaExr0AAAAASUVORK5CYII='); + background-repeat: repeat-y; + background-position: left top; + background-color: #ffcfbb; +} + +.coverLegendCov, .coverLegendNoCov { + padding: 0em 1em 0em 1em; +} + +.headerItem, .headerValue, .headerValueLeg { + white-space: nowrap; +} + +.headerItem { + text-align: right; + font-weight: bold; +} + +.ruler { + background-color: #d9d9d9; +} + +.detail { + font-size: 80%; +} + +.versionInfo { + font-size: 80%; + text-align: right; +} + diff --git a/tests/docker/run.sh b/tests/docker/run.sh index 5c4a877cb..ae30c9784 100644 --- a/tests/docker/run.sh +++ b/tests/docker/run.sh @@ -31,7 +31,7 @@ rust-profdata merge -sparse /app/pgcat-*.profraw -o /app/pgcat.profdata bash -c "rust-cov export -ignore-filename-regex='rustc|registry' -Xdemangler=rustfilt -instr-profile=/app/pgcat.profdata $TEST_OBJECTS --object ./target/debug/pgcat --format lcov > ./lcov.info" -genhtml lcov.info -show-details --highlight --ignore-errors source --legend --output-directory cov --prefix $(pwd) +genhtml lcov.info --title "PgCat Code Coverage" --css-file ./cov-style.css --highlight --no-function-coverage --ignore-errors source --legend --output-directory cov --prefix $(pwd) rm /app/*.profraw rm /app/pgcat.profdata From d193106f812a3d6b8adceede3830f4127be1c4e0 Mon Sep 17 00:00:00 2001 From: Nicholas Dujay <3258756+dat2@users.noreply.github.com> Date: Tue, 21 Feb 2023 16:08:43 -0500 Subject: [PATCH 333/551] implement show users (#329) * implement show users * fix compile errors * add basic ruby test * gitignore things --- .gitignore | 5 ++++- src/admin.rs | 33 +++++++++++++++++++++++++++++++++ tests/ruby/admin_spec.rb | 10 ++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3c6545390..cf28bffe3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ .idea /target *.deb -.vscode \ No newline at end of file +.vscode +.profraw +cov/ +lcov.info diff --git a/src/admin.rs b/src/admin.rs index 71d3e486d..406b0fe01 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -100,6 +100,10 @@ where trace!("SHOW VERSION"); show_version(stream).await } + "USERS" => { + trace!("SHOW USERS"); + show_users(stream).await + } _ => error_response(stream, "Unsupported SHOW query against the admin database").await, }, _ => error_response(stream, "Unsupported query against the admin database").await, @@ -666,3 +670,32 @@ where } } } + +/// Show Users. +async fn show_users(stream: &mut T) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let mut res = BytesMut::new(); + + res.put(row_description(&vec![ + ("name", DataType::Text), + ("pool_mode", DataType::Text), + ])); + + for (user_pool, pool) in get_all_pools() { + let pool_config = &pool.settings; + res.put(data_row(&vec![ + user_pool.user.clone(), + pool_config.pool_mode.to_string(), + ])); + } + + res.put(command_complete("SHOW")); + + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await +} diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index 2000e5d96..783641567 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -286,4 +286,14 @@ connections.map(&:close) end end + + describe "SHOW users" do + it "returns the right users" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW USERS")[0] + admin_conn.close + expect(results["name"]).to eq("sharding_user") + expect(results["pool_mode"]).to eq("transaction") + end + end end From 4d8e0533963e484692ddb874015c24c15beed6fc Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 21 Feb 2023 21:53:10 -0600 Subject: [PATCH 334/551] Fix Back-and-forth RELOAD Bug (#330) We identified a bug where RELOAD fails to update the pools. To reproduce you need to start at some config state, modify that state a bit, reload, revert the configs back to the original state, and reload. The last reload will fail to update the pool because PgCat "thinks" the pool state didn't change. This is because we use a HashSet to keep track of config hashes but we never remove values from it. Say we start with State A, we modify pool configs to State B and reload. Now the POOL_HASHES struct has State A and State B. Attempting to go back to State A will encounter a hashset hit which is interpreted by PgCat as "Configs are the same, no need to reload pools" We fix this by attaching a config_hash value to ConnectionPool object and we calculate that value when we create the pool. This eliminates the need for a global variable. One shortcoming here is that changing any config under one user in the pool will trigger a reload for the entire pool (which is fine I think) --- src/config.rs | 9 +++++- src/pool.rs | 30 +++++++++--------- tests/ruby/helpers/pgcat_process.rb | 2 +- tests/ruby/load_balancing_spec.rb | 2 +- tests/ruby/misc_spec.rb | 49 +++++++++++++++++++++++++++++ 5 files changed, 75 insertions(+), 17 deletions(-) diff --git a/src/config.rs b/src/config.rs index f911d415a..2ed7aebde 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,8 +4,9 @@ use log::{error, info}; use once_cell::sync::Lazy; use regex::Regex; use serde_derive::{Deserialize, Serialize}; +use std::collections::hash_map::DefaultHasher; use std::collections::{BTreeMap, HashMap, HashSet}; -use std::hash::Hash; +use std::hash::{Hash, Hasher}; use std::path::Path; use std::sync::Arc; use tokio::fs::File; @@ -355,6 +356,12 @@ pub struct Pool { } impl Pool { + pub fn hash_value(&self) -> u64 { + let mut s = DefaultHasher::new(); + self.hash(&mut s); + s.finish() + } + pub fn default_pool_mode() -> PoolMode { PoolMode::Transaction } diff --git a/src/pool.rs b/src/pool.rs index cbe5b5007..cca95776f 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -9,7 +9,7 @@ use parking_lot::{Mutex, RwLock}; use rand::seq::SliceRandom; use rand::thread_rng; use regex::Regex; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -37,8 +37,6 @@ pub type PoolMap = HashMap; /// This is atomic and safe and read-optimized. /// The pool is recreated dynamically when the config is reloaded. pub static POOLS: Lazy> = Lazy::new(|| ArcSwap::from_pointee(HashMap::default())); -static POOLS_HASH: Lazy>> = - Lazy::new(|| ArcSwap::from_pointee(HashSet::default())); /// An identifier for a PgCat pool, /// a database visible to clients. @@ -168,6 +166,11 @@ pub struct ConnectionPool { /// to use it. validated: Arc, + /// Hash value for the pool configs. It is used to compare new configs + /// against current config to decide whether or not we need to recreate + /// the pool after a RELOAD command + pub config_hash: u64, + /// If the pool has been paused or not. paused: Arc, paused_waiter: Arc, @@ -181,18 +184,18 @@ impl ConnectionPool { let mut new_pools = HashMap::new(); let mut address_id = 0; - let mut pools_hash = (*(*POOLS_HASH.load())).clone(); - for (pool_name, pool_config) in &config.pools { - let changed = pools_hash.insert(pool_config.clone()); + let new_pool_hash_value = pool_config.hash_value(); // There is one pool per database/user pair. for user in pool_config.users.values() { - // If the pool hasn't changed, get existing reference and insert it into the new_pools. - // We replace all pools at the end, but if the reference is kept, the pool won't get re-created (bb8). - if !changed { - match get_pool(pool_name, &user.username) { - Some(pool) => { + let old_pool_ref = get_pool(pool_name, &user.username); + + match old_pool_ref { + Some(pool) => { + // If the pool hasn't changed, get existing reference and insert it into the new_pools. + // We replace all pools at the end, but if the reference is kept, the pool won't get re-created (bb8). + if pool.config_hash == new_pool_hash_value { info!( "[pool: {}][user: {}] has not changed", pool_name, user.username @@ -203,8 +206,8 @@ impl ConnectionPool { ); continue; } - None => (), } + None => (), } info!( @@ -293,6 +296,7 @@ impl ConnectionPool { addresses, banlist: Arc::new(RwLock::new(banlist)), stats: get_reporter(), + config_hash: new_pool_hash_value, server_info: Arc::new(RwLock::new(BytesMut::new())), settings: PoolSettings { pool_mode: pool_config.pool_mode, @@ -342,8 +346,6 @@ impl ConnectionPool { } POOLS.store(Arc::new(new_pools.clone())); - POOLS_HASH.store(Arc::new(pools_hash.clone())); - Ok(()) } diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index 63b510452..b67c4a86b 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -53,7 +53,7 @@ def current_config def reload_config `kill -s HUP #{@pid}` - sleep 0.1 + sleep 0.5 end def start diff --git a/tests/ruby/load_balancing_spec.rb b/tests/ruby/load_balancing_spec.rb index 9c204c392..e7b89ee8d 100644 --- a/tests/ruby/load_balancing_spec.rb +++ b/tests/ruby/load_balancing_spec.rb @@ -150,7 +150,7 @@ end end - expect(failed_count).to eq(2) + expect(failed_count).to be <= 2 processes.all_databases.each do |instance| queries_routed = instance.count_select_1_plus_2 if processes.replicas[0..1].include?(instance) diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index be7af42b5..2f69fb47d 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -8,6 +8,55 @@ processes.pgcat.shutdown end + context "when adding then removing instance using RELOAD" do + it "works correctly" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + + current_configs = processes.pgcat.current_config + correct_count = current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"].count + expect(admin_conn.async_exec("SHOW DATABASES").count).to eq(correct_count) + + extra_replica = current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"].last.clone + extra_replica[0] = "127.0.0.1" + current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"] << extra_replica + + processes.pgcat.update_config(current_configs) # with replica added + processes.pgcat.reload_config + correct_count = current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"].count + expect(admin_conn.async_exec("SHOW DATABASES").count).to eq(correct_count) + + current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"].pop + + processes.pgcat.update_config(current_configs) # with replica removed again + processes.pgcat.reload_config + correct_count = current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"].count + expect(admin_conn.async_exec("SHOW DATABASES").count).to eq(correct_count) + end + end + + context "when removing then adding instance back using RELOAD" do + it "works correctly" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + + current_configs = processes.pgcat.current_config + correct_count = current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"].count + expect(admin_conn.async_exec("SHOW DATABASES").count).to eq(correct_count) + + removed_replica = current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"].pop + processes.pgcat.update_config(current_configs) # with replica removed + processes.pgcat.reload_config + correct_count = current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"].count + expect(admin_conn.async_exec("SHOW DATABASES").count).to eq(correct_count) + + current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"] << removed_replica + + processes.pgcat.update_config(current_configs) # with replica added again + processes.pgcat.reload_config + correct_count = current_configs["pools"]["sharded_db"]["shards"]["0"]["servers"].count + expect(admin_conn.async_exec("SHOW DATABASES").count).to eq(correct_count) + end + end + describe "TCP Keepalives" do # Ideally, we should block TCP traffic to the database using # iptables to mimic passive (connection is dropped without a RST packet) From a69c07301566420b55514be1882f927a45578b44 Mon Sep 17 00:00:00 2001 From: zainkabani <77307340+zainkabani@users.noreply.github.com> Date: Mon, 27 Feb 2023 01:16:30 -0500 Subject: [PATCH 335/551] Adds SHUTDOWN command as alternate option to sending SIGINT (#331) * Adds SHUTDOWN command to PgCat as alternate option to sending SIGINT * Check if we're already in SHUTDOWN sequence * Send signal directly from shutdown instead of using channel * Add tests * trigger build * Lowercase response and boolean change * Update tests * Fix tests * typo --- Cargo.lock | 245 +++++++++++++++++++++++++----------------- Cargo.toml | 1 + src/admin.rs | 36 ++++++- src/main.rs | 150 +++++++++++++------------- tests/python/tests.py | 61 +++++++++++ 5 files changed, 324 insertions(+), 169 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b18b2c1e..41b32ff44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,9 +79,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bytes" @@ -91,9 +91,9 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -153,9 +153,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.86" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" +checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" dependencies = [ "cc", "cxxbridge-flags", @@ -165,9 +165,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.86" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" +checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" dependencies = [ "cc", "codespan-reporting", @@ -180,15 +180,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.86" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" +checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" [[package]] name = "cxxbridge-macro" -version = "1.0.86" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" +checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ "proc-macro2", "quote", @@ -252,12 +252,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "fs_extra" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" - [[package]] name = "futures" version = "0.3.26" @@ -402,6 +396,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hmac" version = "0.12.1" @@ -413,9 +413,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -511,24 +511,24 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -539,12 +539,11 @@ checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "jemalloc-sys" -version = "0.5.2+5.3.0-patched" +version = "0.5.3+5.3.0-patched" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134163979b6eed9564c98637b710b40979939ba351f59952708234ea11b5f3f8" +checksum = "f9bd5d616ea7ed58b571b2e209a65759664d7fb021a0819d7a790afc67e47ca1" dependencies = [ "cc", - "fs_extra", "libc", ] @@ -560,9 +559,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -622,25 +621,39 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] -name = "nom8" -version = "0.2.0" +name = "nix" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "memchr", + "bitflags", + "cfg-if", + "libc", + "memoffset", + "pin-utils", + "static_assertions", ] [[package]] @@ -668,7 +681,7 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -690,15 +703,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.5" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -719,6 +732,7 @@ dependencies = [ "jemallocator", "log", "md-5", + "nix", "num_cpus", "once_cell", "parking_lot", @@ -800,9 +814,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] @@ -889,23 +903,23 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.6" +version = "0.36.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "rustls" -version = "0.20.7" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", @@ -994,9 +1008,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] @@ -1009,9 +1023,9 @@ checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -1047,6 +1061,12 @@ dependencies = [ "log", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stringprep" version = "0.1.2" @@ -1065,9 +1085,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", @@ -1076,9 +1096,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -1105,9 +1125,9 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" @@ -1126,7 +1146,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1153,9 +1173,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -1167,9 +1187,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772c1426ab886e7362aedf4abc9c0d1348a979517efedfc25862944d10137af0" +checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6" dependencies = [ "serde", "serde_spanned", @@ -1188,15 +1208,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.1" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90a238ee2e6ede22fb95350acc78e21dc40da00bb66c0334bde83de4ed89424e" +checksum = "9a1eb0622d28f4b9c90adc4ea4b2b46b47663fde9ac5fafcb14a1369d5508825" dependencies = [ "indexmap", - "nom8", "serde", "serde_spanned", "toml_datetime", + "winnow", ] [[package]] @@ -1239,9 +1259,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" @@ -1300,9 +1320,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1310,9 +1330,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -1325,9 +1345,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1335,9 +1355,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -1348,15 +1368,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -1418,44 +1438,77 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" + +[[package]] +name = "winnow" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658" +dependencies = [ + "memchr", +] diff --git a/Cargo.toml b/Cargo.toml index 3cf0e7aa4..0c0202754 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ phf = { version = "0.11.1", features = ["macros"] } exitcode = "1.1.2" futures = "0.3" socket2 = { version = "0.4.7", features = ["all"] } +nix = "0.26.2" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/src/admin.rs b/src/admin.rs index 406b0fe01..7f7ada565 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,6 +1,8 @@ /// Admin database. use bytes::{Buf, BufMut, BytesMut}; -use log::{info, trace}; +use log::{error, info, trace}; +use nix::sys::signal::{self, Signal}; +use nix::unistd::Pid; use std::collections::HashMap; use tokio::time::Instant; @@ -67,6 +69,10 @@ where trace!("RESUME"); resume(stream, query_parts[1]).await } + "SHUTDOWN" => { + trace!("SHUTDOWN"); + shutdown(stream).await + } "SHOW" => match query_parts[1].to_ascii_uppercase().as_str() { "CONFIG" => { trace!("SHOW CONFIG"); @@ -671,6 +677,34 @@ where } } +/// Send response packets for shutdown. +async fn shutdown(stream: &mut T) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let mut res = BytesMut::new(); + + res.put(row_description(&vec![("success", DataType::Text)])); + + let mut shutdown_success = "t"; + + let pid = std::process::id(); + if signal::kill(Pid::from_raw(pid.try_into().unwrap()), Signal::SIGINT).is_err() { + error!("Unable to send SIGINT to PID: {}", pid); + shutdown_success = "f"; + } + + res.put(data_row(&vec![shutdown_success.to_string()])); + + res.put(command_complete("SHUTDOWN")); + + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await +} + /// Show Users. async fn show_users(stream: &mut T) -> Result<(), Error> where diff --git a/src/main.rs b/src/main.rs index cd408c491..33536b0f6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -232,7 +232,13 @@ fn main() -> Result<(), Box> { // Initiate graceful shutdown sequence on sig int _ = interrupt_signal.recv() => { - info!("Got SIGINT, waiting for client connection drain now"); + info!("Got SIGINT"); + + // Don't want this to happen more than once + if admin_only { + continue; + } + admin_only = true; // Broadcast that client tasks need to finish @@ -241,98 +247,98 @@ fn main() -> Result<(), Box> { let _ = drain_tx.send(0).await; tokio::task::spawn(async move { - let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(config.general.shutdown_timeout)); + let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(config.general.shutdown_timeout)); - // First tick fires immediately. - interval.tick().await; + // First tick fires immediately. + interval.tick().await; - // Second one in the interval time. - interval.tick().await; + // Second one in the interval time. + interval.tick().await; - // We're done waiting. - error!("Graceful shutdown timed out. {} active clients being closed", total_clients); + // We're done waiting. + error!("Graceful shutdown timed out. {} active clients being closed", total_clients); - let _ = exit_tx.send(()).await; + let _ = exit_tx.send(()).await; }); }, _ = term_signal.recv() => { info!("Got SIGTERM, closing with {} clients active", total_clients); break; - }, + }, - new_client = listener.accept() => { - let (socket, addr) = match new_client { - Ok((socket, addr)) => (socket, addr), - Err(err) => { - error!("{:?}", err); - continue; - } - }; - - let shutdown_rx = shutdown_tx.subscribe(); - let drain_tx = drain_tx.clone(); - let client_server_map = client_server_map.clone(); - - let tls_certificate = config.general.tls_certificate.clone(); - - tokio::task::spawn(async move { - let start = chrono::offset::Utc::now().naive_utc(); - - match client::client_entrypoint( - socket, - client_server_map, - shutdown_rx, - drain_tx, - admin_only, - tls_certificate.clone(), - config.general.log_client_connections, - ) - .await - { - Ok(()) => { - let duration = chrono::offset::Utc::now().naive_utc() - start; - - if config.general.log_client_disconnections { - info!( - "Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); - } else { - debug!( - "Client {:?} disconnected, session duration: {}", - addr, - format_duration(&duration) - ); - } + new_client = listener.accept() => { + let (socket, addr) = match new_client { + Ok((socket, addr)) => (socket, addr), + Err(err) => { + error!("{:?}", err); + continue; } + }; - Err(err) => { - match err { - errors::Error::ClientBadStartup => debug!("Client disconnected with error {:?}", err), - _ => warn!("Client disconnected with error {:?}", err), + let shutdown_rx = shutdown_tx.subscribe(); + let drain_tx = drain_tx.clone(); + let client_server_map = client_server_map.clone(); + + let tls_certificate = config.general.tls_certificate.clone(); + + tokio::task::spawn(async move { + let start = chrono::offset::Utc::now().naive_utc(); + + match client::client_entrypoint( + socket, + client_server_map, + shutdown_rx, + drain_tx, + admin_only, + tls_certificate.clone(), + config.general.log_client_connections, + ) + .await + { + Ok(()) => { + let duration = chrono::offset::Utc::now().naive_utc() - start; + + if config.general.log_client_disconnections { + info!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + } else { + debug!( + "Client {:?} disconnected, session duration: {}", + addr, + format_duration(&duration) + ); + } } - } - }; - }); - } + Err(err) => { + match err { + errors::Error::ClientBadStartup => debug!("Client disconnected with error {:?}", err), + _ => warn!("Client disconnected with error {:?}", err), + } - _ = exit_rx.recv() => { - break; - } + } + }; + }); + } + + _ = exit_rx.recv() => { + break; + } - client_ping = drain_rx.recv() => { - let client_ping = client_ping.unwrap(); - total_clients += client_ping; + client_ping = drain_rx.recv() => { + let client_ping = client_ping.unwrap(); + total_clients += client_ping; - if total_clients == 0 && admin_only { - let _ = exit_tx.send(()).await; + if total_clients == 0 && admin_only { + let _ = exit_tx.send(()).await; + } } } } - } info!("Shutting down..."); }); diff --git a/tests/python/tests.py b/tests/python/tests.py index 7c9907988..6108ff218 100644 --- a/tests/python/tests.py +++ b/tests/python/tests.py @@ -110,6 +110,37 @@ def test_shutdown_logic(): cleanup_conn(conn, cur) pg_cat_send_signal(signal.SIGTERM) + # - - - - - - - - - - - - - - - - - - + # NO ACTIVE QUERIES ADMIN SHUTDOWN COMMAND + + # Start pgcat + pgcat_start() + + # Create client connection and begin transaction + conn, cur = connect_db() + admin_conn, admin_cur = connect_db(admin=True) + + cur.execute("BEGIN;") + cur.execute("SELECT 1;") + cur.execute("COMMIT;") + + # Send SHUTDOWN command pgcat while not in transaction + admin_cur.execute("SHUTDOWN;") + time.sleep(1) + + # Check that any new queries fail after SHUTDOWN command since server should close with no active transactions + try: + cur.execute("SELECT 1;") + except psycopg2.OperationalError as e: + pass + else: + # Fail if query execution succeeded + raise Exception("Server not closed after sigint") + + cleanup_conn(conn, cur) + cleanup_conn(admin_conn, admin_cur) + pg_cat_send_signal(signal.SIGTERM) + # - - - - - - - - - - - - - - - - - - # HANDLE TRANSACTION WITH SIGINT @@ -136,6 +167,36 @@ def test_shutdown_logic(): cleanup_conn(conn, cur) pg_cat_send_signal(signal.SIGTERM) + # - - - - - - - - - - - - - - - - - - + # HANDLE TRANSACTION WITH ADMIN SHUTDOWN COMMAND + + # Start pgcat + pgcat_start() + + # Create client connection and begin transaction + conn, cur = connect_db() + admin_conn, admin_cur = connect_db(admin=True) + + cur.execute("BEGIN;") + cur.execute("SELECT 1;") + + # Send SHUTDOWN command pgcat while still in transaction + admin_cur.execute("SHUTDOWN;") + if admin_cur.fetchall()[0][0] != "t": + raise Exception("PgCat unable to send signal") + time.sleep(1) + + # Check that any new queries succeed after SHUTDOWN command since server should still allow transaction to complete + try: + cur.execute("SELECT 1;") + except psycopg2.OperationalError as e: + # Fail if query fails since server closed + raise Exception("Server closed while in transaction", e.pgerror) + + cleanup_conn(conn, cur) + cleanup_conn(admin_conn, admin_cur) + pg_cat_send_signal(signal.SIGTERM) + # - - - - - - - - - - - - - - - - - - # NO NEW NON-ADMIN CONNECTIONS DURING SHUTDOWN # Start pgcat From d6c5656d821b5c972fb1dcebcd160ba97611de49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Tue, 28 Feb 2023 22:10:40 +0100 Subject: [PATCH 336/551] Allow sending logs to stdout by using STDOUT_LOG env var (#334) * Allow sending logs to stdout by using STDOUT_LOG env var * Increase stats buffer size --- src/lib.rs | 1 + src/main.rs | 5 +-- src/multi_logger.rs | 80 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 src/multi_logger.rs diff --git a/src/lib.rs b/src/lib.rs index e9a683f3d..63eae59b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ pub mod config; pub mod constants; pub mod errors; pub mod messages; +pub mod multi_logger; pub mod pool; pub mod scram; pub mod server; diff --git a/src/main.rs b/src/main.rs index 33536b0f6..b3ef77c30 100644 --- a/src/main.rs +++ b/src/main.rs @@ -66,6 +66,7 @@ mod config; mod constants; mod errors; mod messages; +mod multi_logger; mod pool; mod prometheus; mod query_router; @@ -81,7 +82,7 @@ use crate::prometheus::start_metric_server; use crate::stats::{Collector, Reporter, REPORTER}; fn main() -> Result<(), Box> { - env_logger::builder().format_timestamp_micros().init(); + multi_logger::MultiLogger::init().unwrap(); info!("Welcome to PgCat! Meow. (Version {})", VERSION); @@ -160,7 +161,7 @@ fn main() -> Result<(), Box> { let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); // Statistics reporting. - let (stats_tx, stats_rx) = mpsc::channel(100_000); + let (stats_tx, stats_rx) = mpsc::channel(500_000); REPORTER.store(Arc::new(Reporter::new(stats_tx.clone()))); // Connection pool that allows to query all shards and replicas. diff --git a/src/multi_logger.rs b/src/multi_logger.rs new file mode 100644 index 000000000..901db02e5 --- /dev/null +++ b/src/multi_logger.rs @@ -0,0 +1,80 @@ +use log::{Level, Log, Metadata, Record, SetLoggerError}; + +// This is a special kind of logger that allows sending logs to different +// targets depending on the log level. +// +// By default, if nothing is set, it acts as a regular env_log logger, +// it sends everything to standard error. +// +// If the Env variable `STDOUT_LOG` is defined, it will be used for +// configuring the standard out logger. +// +// The behavior is: +// - If it is an error, the message is written to standard error. +// - If it is not, and it matches the log level of the standard output logger (`STDOUT_LOG` env var), it will be send to standard output. +// - If the above is not true, it is sent to the stderr logger that will log it or not depending on the value +// of the RUST_LOG env var. +// +// So to summarize, if no `STDOUT_LOG` env var is present, the logger is the default logger. If `STDOUT_LOG` is set, everything +// but errors, that matches the log level set in the `STDOUT_LOG` env var is sent to stdout. You can have also some esoteric configuration +// where you set `RUST_LOG=debug` and `STDOUT_LOG=info`, in here, erros will go to stderr, warns and infos to stdout and debugs to stderr. +// +pub struct MultiLogger { + stderr_logger: env_logger::Logger, + stdout_logger: env_logger::Logger, +} + +impl MultiLogger { + fn new() -> Self { + let stderr_logger = env_logger::builder().format_timestamp_micros().build(); + let stdout_logger = env_logger::Builder::from_env("STDOUT_LOG") + .format_timestamp_micros() + .target(env_logger::Target::Stdout) + .build(); + + Self { + stderr_logger, + stdout_logger, + } + } + + pub fn init() -> Result<(), SetLoggerError> { + let logger = Self::new(); + + log::set_max_level(logger.stderr_logger.filter()); + log::set_boxed_logger(Box::new(logger)) + } +} + +impl Log for MultiLogger { + fn enabled(&self, metadata: &Metadata) -> bool { + self.stderr_logger.enabled(metadata) && self.stdout_logger.enabled(metadata) + } + + fn log(&self, record: &Record) { + if record.level() == Level::Error { + self.stderr_logger.log(record); + } else { + if self.stdout_logger.matches(record) { + self.stdout_logger.log(record); + } else { + self.stderr_logger.log(record); + } + } + } + + fn flush(&self) { + self.stderr_logger.flush(); + self.stdout_logger.flush(); + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_init() { + MultiLogger::init().unwrap(); + } +} From 2e71f751b30d854f027b1f3186774b8ad2a62eb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:40:52 -0500 Subject: [PATCH 337/551] chore(deps): bump sqlparser from 0.30.0 to 0.31.0 (#335) Bumps [sqlparser](https://github.com/sqlparser-rs/sqlparser-rs) from 0.30.0 to 0.31.0. - [Release notes](https://github.com/sqlparser-rs/sqlparser-rs/releases) - [Changelog](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/sqlparser-rs/sqlparser-rs/compare/v0.30.0...v0.31.0) --- updated-dependencies: - dependency-name: sqlparser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 41b32ff44..4601e576e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1054,9 +1054,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db67dc6ef36edb658196c3fef0464a80b53dbbc194a904e81f9bd4190f9ecc5b" +checksum = "f064eb7b163863163c29801910f763c6bfb563b8a8ca4c54193da4c1eea57547" dependencies = [ "log", ] diff --git a/Cargo.toml b/Cargo.toml index 0c0202754..9ea45b3d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = "0.30.0" +sqlparser = "0.31.0" log = "0.4" arc-swap = "1" env_logger = "0.10" From a3434817824c1176df270c96e4ed46f0712632a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:41:33 -0500 Subject: [PATCH 338/551] chore(deps): bump tokio from 1.25.0 to 1.26.0 (#336) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.25.0 to 1.26.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.25.0...tokio-1.26.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4601e576e..34850206e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -516,7 +516,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -528,7 +528,7 @@ dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -639,7 +639,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -711,7 +711,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -912,7 +912,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -1131,9 +1131,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -1146,7 +1146,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -1423,21 +1423,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - [[package]] name = "windows-sys" version = "0.45.0" From 388f6706c867d0b663be0e8688e8ce2ac7f8e52c Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 2 Mar 2023 00:53:31 -0500 Subject: [PATCH 339/551] Automatic sharding for SELECT v2 (#337) * More comprehensive read sharding support * A few fixes * fq * comment * wildcard --- pgcat.toml | 2 +- src/config.rs | 21 ++++- src/query_router.rs | 197 ++++++++++++++++++++++++++++++++++++++------ 3 files changed, 192 insertions(+), 28 deletions(-) diff --git a/pgcat.toml b/pgcat.toml index 26f9d7ddc..b5328b643 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -102,7 +102,7 @@ primary_reads_enabled = true sharding_function = "pg_bigint_hash" # Automatically parse this from queries and route queries to the right shard! -automatic_sharding_key = "id" +automatic_sharding_key = "data.id" # Idle timeout can be overwritten in the pool idle_timeout = 40000 diff --git a/src/config.rs b/src/config.rs index 2ed7aebde..517cabce1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -374,7 +374,7 @@ impl Pool { None } - pub fn validate(&self) -> Result<(), Error> { + pub fn validate(&mut self) -> Result<(), Error> { match self.default_role.as_ref() { "any" => (), "primary" => (), @@ -414,6 +414,25 @@ impl Pool { } } + self.automatic_sharding_key = match &self.automatic_sharding_key { + Some(key) => { + // No quotes in the key so we don't have to compare quoted + // to unquoted idents. + let key = key.replace("\"", ""); + + if key.split(".").count() != 2 { + error!( + "automatic_sharding_key '{}' must be fully qualified, e.g. t.{}`", + key, key + ); + return Err(Error::BadConfig); + } + + Some(key) + } + None => None, + }; + Ok(()) } } diff --git a/src/query_router.rs b/src/query_router.rs index bf07db7fe..fff5bba8d 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -5,7 +5,9 @@ use log::{debug, error}; use once_cell::sync::OnceCell; use regex::{Regex, RegexSet}; use sqlparser::ast::Statement::{Query, StartTransaction}; -use sqlparser::ast::{BinaryOperator, Expr, SetExpr, Value}; +use sqlparser::ast::{ + BinaryOperator, Expr, Ident, JoinConstraint, JoinOperator, SetExpr, TableFactor, Value, +}; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; @@ -403,20 +405,67 @@ impl QueryRouter { /// A `selection` is the `WHERE` clause. This parses /// the clause and extracts the sharding key, if present. - fn selection_parser(&self, expr: &Expr) -> Vec { + fn selection_parser(&self, expr: &Expr, table_names: &Vec>) -> Vec { let mut result = Vec::new(); let mut found = false; + let sharding_key = self + .pool_settings + .automatic_sharding_key + .as_ref() + .unwrap() + .split(".") + .map(|ident| Ident::new(ident)) + .collect::>(); + + // Sharding key must be always fully qualified + assert_eq!(sharding_key.len(), 2); + // This parses `sharding_key = 5`. But it's technically // legal to write `5 = sharding_key`. I don't judge the people // who do that, but I think ORMs will still use the first variant, // so we can leave the second as a TODO. if let Expr::BinaryOp { left, op, right } = expr { match &**left { - Expr::BinaryOp { .. } => result.extend(self.selection_parser(left)), + Expr::BinaryOp { .. } => result.extend(self.selection_parser(left, table_names)), Expr::Identifier(ident) => { - found = - ident.value == *self.pool_settings.automatic_sharding_key.as_ref().unwrap(); + // Only if we're dealing with only one table + // and there is no ambiguity + if &ident.value == &sharding_key[1].value { + // Sharding key is unique enough, don't worry about + // table names. + if &sharding_key[0].value == "*" { + found = true; + } else if table_names.len() == 1 { + let table = &table_names[0]; + + if table.len() == 1 { + // Table is not fully qualified, e.g. + // SELECT * FROM t WHERE sharding_key = 5 + // Make sure the table name from the sharding key matches + // the table name from the query. + found = &sharding_key[0].value == &table[0].value; + } else if table.len() == 2 { + // Table name is fully qualified with the schema: e.g. + // SELECT * FROM public.t WHERE sharding_key = 5 + // Ignore the schema (TODO: at some point, we want schema support) + // and use the table name only. + found = &sharding_key[0].value == &table[1].value; + } else { + debug!("Got table name with more than two idents, which is not possible"); + } + } + } + } + + Expr::CompoundIdentifier(idents) => { + // The key is fully qualified in the query, + // it will exist or Postgres will throw an error. + if idents.len() == 2 { + found = &sharding_key[0].value == &idents[0].value + && &sharding_key[1].value == &idents[1].value; + } + // TODO: key can have schema as well, e.g. public.data.id (len == 3) } _ => (), }; @@ -433,7 +482,7 @@ impl QueryRouter { }; match &**right { - Expr::BinaryOp { .. } => result.extend(self.selection_parser(right)), + Expr::BinaryOp { .. } => result.extend(self.selection_parser(right, table_names)), Expr::Value(Value::Number(value, ..)) => { if found { match value.parse::() { @@ -456,6 +505,7 @@ impl QueryRouter { /// Try to figure out which shard the query should go to. fn infer_shard(&self, query: &sqlparser::ast::Query) -> Option { let mut shards = BTreeSet::new(); + let mut exprs = Vec::new(); match &*query.body { SetExpr::Query(query) => { @@ -467,27 +517,75 @@ impl QueryRouter { }; } + // SELECT * FROM ... + // We understand that pretty well. SetExpr::Select(select) => { - match &select.selection { - Some(selection) => { - let sharding_keys = self.selection_parser(selection); + // Collect all table names from the query. + let mut table_names = Vec::new(); - // TODO: Add support for prepared statements here. - // This should just give us the position of the value in the `B` message. + for table in select.from.iter() { + match &table.relation { + TableFactor::Table { name, .. } => { + table_names.push(name.0.clone()); + } - let sharder = Sharder::new( - self.pool_settings.shards, - self.pool_settings.sharding_function, - ); + _ => (), + }; - for value in sharding_keys { - let shard = sharder.shard(value); - shards.insert(shard); - } + // Get table names from all the joins. + for join in table.joins.iter() { + match &join.relation { + TableFactor::Table { name, .. } => { + table_names.push(name.0.clone()); + } + + _ => (), + }; + + // We can filter results based on join conditions, e.g. + // SELECT * FROM t INNER JOIN B ON B.sharding_key = 5; + match &join.join_operator { + JoinOperator::Inner(inner_join) => match &inner_join { + JoinConstraint::On(expr) => { + // Parse the selection criteria later. + exprs.push(expr.clone()); + } + + _ => (), + }, + + _ => (), + }; + } + } + + // Parse the actual "FROM ..." + match &select.selection { + Some(selection) => { + exprs.push(selection.clone()); } None => (), }; + + // Look for sharding keys in either the join condition + // or the selection. + for expr in exprs.iter() { + let sharding_keys = self.selection_parser(expr, &table_names); + + // TODO: Add support for prepared statements here. + // This should just give us the position of the value in the `B` message. + + let sharder = Sharder::new( + self.pool_settings.shards, + self.pool_settings.sharding_function, + ); + + for value in sharding_keys { + let shard = sharder.shard(value); + shards.insert(shard); + } + } } _ => (), }; @@ -825,7 +923,7 @@ mod test { query_parser_enabled: true, primary_reads_enabled: false, sharding_function: ShardingFunction::PgBigintHash, - automatic_sharding_key: Some(String::from("id")), + automatic_sharding_key: Some(String::from("test.id")), healthcheck_delay: PoolSettings::default().healthcheck_delay, healthcheck_timeout: PoolSettings::default().healthcheck_timeout, ban_time: PoolSettings::default().ban_time, @@ -854,11 +952,6 @@ mod test { let q2 = simple_query("SET SERVER ROLE TO 'default'"); assert!(qr.try_execute_command(&q2) != None); assert_eq!(qr.active_role.unwrap(), pool_settings.default_role); - - // Here we go :) - let q3 = simple_query("SELECT * FROM test WHERE id = 5 AND values IN (1, 2, 3)"); - assert!(qr.infer(&q3)); - assert_eq!(qr.shard(), 1); } #[test] @@ -891,7 +984,7 @@ mod test { query_parser_enabled: true, primary_reads_enabled: false, sharding_function: ShardingFunction::PgBigintHash, - automatic_sharding_key: Some(String::from("id")), + automatic_sharding_key: None, healthcheck_delay: PoolSettings::default().healthcheck_delay, healthcheck_timeout: PoolSettings::default().healthcheck_timeout, ban_time: PoolSettings::default().ban_time, @@ -920,4 +1013,56 @@ mod test { assert!(qr.try_execute_command(&q2) == None); assert_eq!(qr.active_shard, Some(2)); } + + #[test] + fn test_automatic_sharding_key() { + QueryRouter::setup(); + + let mut qr = QueryRouter::new(); + qr.pool_settings.automatic_sharding_key = Some("data.id".to_string()); + qr.pool_settings.shards = 3; + + assert!(qr.infer(&simple_query("SELECT * FROM data WHERE id = 5"))); + assert_eq!(qr.shard(), 2); + + assert!(qr.infer(&simple_query( + "SELECT one, two, three FROM public.data WHERE id = 6" + ))); + assert_eq!(qr.shard(), 0); + + assert!(qr.infer(&simple_query( + "SELECT * FROM data + INNER JOIN t2 ON data.id = 5 + AND t2.data_id = data.id + WHERE data.id = 5" + ))); + assert_eq!(qr.shard(), 2); + + // Shard did not move because we couldn't determine the sharding key since it could be ambiguous + // in the query. + assert!(qr.infer(&simple_query( + "SELECT * FROM t2 INNER JOIN data ON id = 6 AND data.id = t2.data_id" + ))); + assert_eq!(qr.shard(), 2); + + assert!(qr.infer(&simple_query( + r#"SELECT * FROM "public"."data" WHERE "id" = 6"# + ))); + assert_eq!(qr.shard(), 0); + + assert!(qr.infer(&simple_query( + r#"SELECT * FROM "public"."data" WHERE "data"."id" = 5"# + ))); + assert_eq!(qr.shard(), 2); + + // Super unique sharding key + qr.pool_settings.automatic_sharding_key = Some("*.unique_enough_column_name".to_string()); + assert!(qr.infer(&simple_query( + "SELECT * FROM table_x WHERE unique_enough_column_name = 6" + ))); + assert_eq!(qr.shard(), 0); + + assert!(qr.infer(&simple_query("SELECT * FROM table_y WHERE another_key = 5"))); + assert_eq!(qr.shard(), 0); + } } From ffa08ce93f48e8a94b0d8d5d25c09a95c29d408c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Thu, 2 Mar 2023 18:14:10 +0100 Subject: [PATCH 340/551] Dev environment (#338) Add dev env --- .gitignore | 5 ++ README.md | 11 +++ dev/Dockerfile | 33 ++++++++ dev/dev_bashrc | 120 ++++++++++++++++++++++++++++ dev/docker-compose.yaml | 84 +++++++++++++++++++ dev/script/console | 6 ++ tests/ruby/helpers/pgcat_process.rb | 8 +- 7 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 dev/Dockerfile create mode 100644 dev/dev_bashrc create mode 100644 dev/docker-compose.yaml create mode 100755 dev/script/console diff --git a/.gitignore b/.gitignore index cf28bffe3..b3ca0139d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ .profraw cov/ lcov.info + +# Dev +dev/.bash_history +dev/cache +!dev/cache/.keepme diff --git a/README.md b/README.md index 2ba7ba828..a8723b603 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,17 @@ docker compose up --exit-code-from main # This will also produce coverage report | Statistics | :white_check_mark: | :white_check_mark: | Query the admin database with `psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS'`. | | Live config reloading | :white_check_mark: | :white_check_mark: | Run `kill -s SIGHUP $(pgrep pgcat)` and watch the config reload. | +### Dev + +Also, you can open a 'dev' environment where you can debug tests easier by running the following command: + +``` +./dev/script/console +``` + +This will open a terminal in an environment similar to that used in tests. In there you can compile, run tests, do some debugging with the test environment, etc. Objects +compiled inside the contaner (and bundled gems) will be placed in `dev/cache` so they don't interfere with what you have in your host. + ## Usage ### Session mode diff --git a/dev/Dockerfile b/dev/Dockerfile new file mode 100644 index 000000000..f2054ef2d --- /dev/null +++ b/dev/Dockerfile @@ -0,0 +1,33 @@ +FROM rust:bullseye + +# Dependencies +RUN apt-get update -y \ + && apt-get install -y \ + llvm-11 psmisc postgresql-contrib postgresql-client \ + ruby ruby-dev libpq-dev python3 python3-pip lcov curl sudo iproute2 \ + strace ngrep iproute2 dnsutils lsof net-tools telnet + +# Rust +RUN cargo install cargo-binutils rustfilt +RUN rustup component add llvm-tools-preview + +# Ruby +RUN sudo gem install bundler + +# Toxyproxy +RUN wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ + sudo dpkg -i toxiproxy-2.4.0.deb + +# Config +ENV APP_ROOT=/app +ARG APP_USER=pgcat +COPY dev_bashrc /etc/bash.bashrc + +RUN useradd -m -o -u 999 ${APP_USER} || exit 0 && mkdir ${APP_ROOT} && chown ${APP_USER} ${APP_ROOT} +RUN adduser ${APP_USER} sudo \ + && echo "${APP_USER} ALL=NOPASSWD: ALL" > /etc/sudoers.d/${APP_USER} \ + && chmod ugo+s /usr/sbin/usermod /usr/sbin/groupmod +ENV HOME=${APP_ROOT} +WORKDIR ${APP_ROOT} + +ENTRYPOINT ["/bin/bash"] diff --git a/dev/dev_bashrc b/dev/dev_bashrc new file mode 100644 index 000000000..25d6a03af --- /dev/null +++ b/dev/dev_bashrc @@ -0,0 +1,120 @@ +# ~/.bashrc: executed by bash(1) for non-login shells. +# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc) +# for examples + +# FIX USER NEEDED SO WE CAN SHARE UID BETWEEN HOST AND DEV ENV +usermod -o -u $(id -u) pgcat +groupmod -o -g $(id -g) pgcat + +# We fix the setuid in those commands as we now have sudo +sudo chmod ugo-s /usr/sbin/usermod /usr/sbin/groupmod + +# Environment customization +export DEV_ROOT="${APP_ROOT}/dev" +export HISTFILE="${DEV_ROOT}/.bash_history" +export CARGO_TARGET_DIR="${DEV_ROOT}/cache/target" +export CARGO_HOME="${DEV_ROOT}/cache/target/.cargo" +export BUNDLE_PATH="${DEV_ROOT}/cache/bundle" + +# Regular bashrc +# If not running interactively, don't do anything +case $- in + *i*) ;; + *) return;; +esac + +# don't put duplicate lines or lines starting with space in the history. +# See bash(1) for more options +HISTCONTROL=ignoreboth + +# append to the history file, don't overwrite it +shopt -s histappend + +# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) +HISTSIZE=1000 +HISTFILESIZE=2000 + +# check the window size after each command and, if necessary, +# update the values of LINES and COLUMNS. +shopt -s checkwinsize + +# If set, the pattern "**" used in a pathname expansion context will +# match all files and zero or more directories and subdirectories. +#shopt -s globstar + +# make less more friendly for non-text input files, see lesspipe(1) +[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" + +# set variable identifying the chroot you work in (used in the prompt below) +if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then + debian_chroot=$(cat /etc/debian_chroot) +fi + +# set a fancy prompt (non-color, unless we know we "want" color) +case "$TERM" in + xterm-color|*-256color) color_prompt=yes;; +esac + +# uncomment for a colored prompt, if the terminal has the capability; turned +# off by default to not distract the user: the focus in a terminal window +# should be on the output of commands, not on the prompt +#force_color_prompt=yes + +if [ -n "$force_color_prompt" ]; then + if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then + # We have color support; assume it's compliant with Ecma-48 + # (ISO/IEC-6429). (Lack of such support is extremely rare, and such + # a case would tend to support setf rather than setaf.) + color_prompt=yes + else + color_prompt= + fi +fi + +PS1='\[\e]0;pgcat@dev-container\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]pgcat\[\033[00m\]@\[\033[01;32m\]dev-container\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\[\033[01;31m\]$(git branch &>/dev/null; if [ $? -eq 0 ]; then echo " ($(git branch | grep ^* |sed s/\*\ //))"; fi)\[\033[00m\]\$ ' + +unset color_prompt force_color_prompt + +# enable color support of ls and also add handy aliases +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + alias ls='ls --color=auto' + #alias dir='dir --color=auto' + #alias vdir='vdir --color=auto' + + alias grep='grep --color=auto' + alias fgrep='fgrep --color=auto' + alias egrep='egrep --color=auto' +fi + +# colored GCC warnings and errors +#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01' + +# some more ls aliases +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' + +# Add an "alert" alias for long running commands. Use like so: +# sleep 10; alert +alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"' + +# Alias definitions. +# You may want to put all your additions into a separate file like +# ~/.bash_aliases, instead of adding them here directly. +# See /usr/share/doc/bash-doc/examples in the bash-doc package. + +if [ -f ~/.bash_aliases ]; then + . ~/.bash_aliases +fi + +# enable programmable completion features (you don't need to enable +# this, if it's already enabled in /etc/bash.bashrc and /etc/profile +# sources /etc/bash.bashrc). +if ! shopt -oq posix; then + if [ -f /usr/share/bash-completion/bash_completion ]; then + . /usr/share/bash-completion/bash_completion + elif [ -f /etc/bash_completion ]; then + . /etc/bash_completion + fi +fi diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml new file mode 100644 index 000000000..ee609e0fe --- /dev/null +++ b/dev/docker-compose.yaml @@ -0,0 +1,84 @@ +version: "3" + +x-common-definition-pg: + &common-definition-pg + image: postgres:14 + network_mode: "service:main" + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres -d postgres" ] + interval: 5s + timeout: 5s + retries: 5 + volumes: + - type: bind + source: ../tests/sharding/query_routing_setup.sql + target: /docker-entrypoint-initdb.d/query_routing_setup.sql + - type: bind + source: ../tests/sharding/partition_hash_test_setup.sql + target: /docker-entrypoint-initdb.d/partition_hash_test_setup.sql + +x-common-env-pg: + &common-env-pg + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + +services: + main: + image: kubernetes/pause + + pg1: + <<: *common-definition-pg + environment: + <<: *common-env-pg + POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 + PGPORT: 5432 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "5432"] + + pg2: + <<: *common-definition-pg + environment: + <<: *common-env-pg + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 + PGPORT: 7432 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "7432"] + pg3: + <<: *common-definition-pg + environment: + <<: *common-env-pg + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 + PGPORT: 8432 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "8432"] + pg4: + <<: *common-definition-pg + environment: + <<: *common-env-pg + POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 + PGPORT: 9432 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "9432"] + + toxiproxy: + build: . + network_mode: "service:main" + container_name: toxiproxy + environment: + LOG_LEVEL: info + entrypoint: toxiproxy-server + depends_on: + - pg1 + - pg2 + - pg3 + - pg4 + + pgcat-shell: + stdin_open: true + user: "${HOST_UID}:${HOST_GID}" + build: . + network_mode: "service:main" + depends_on: + - toxiproxy + volumes: + - ../:/app/ + entrypoint: + - /bin/bash + - -i diff --git a/dev/script/console b/dev/script/console new file mode 100755 index 000000000..f2a12d65b --- /dev/null +++ b/dev/script/console @@ -0,0 +1,6 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export HOST_UID="$(id -u)" +export HOST_GID="$(id -g)" +docker-compose -f "${DIR}/../docker-compose.yaml" run --rm pgcat-shell diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index b67c4a86b..2108eafc2 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -24,7 +24,13 @@ def initialize(log_level) @log_filename = "/tmp/pgcat_log_#{SecureRandom.urlsafe_base64}.log" @config_filename = "/tmp/pgcat_cfg_#{SecureRandom.urlsafe_base64}.toml" - @command = "../../target/debug/pgcat #{@config_filename}" + command_path = if ENV['CARGO_TARGET_DIR'] then + "#{ENV['CARGO_TARGET_DIR']}/debug/pgcat" + else + '../../target/debug/pgcat' + end + + @command = "#{command_path} #{@config_filename}" FileUtils.cp("../../pgcat.toml", @config_filename) cfg = current_config From 9a76cb3f8ce464fe59baa1aa025240dd29c02f0e Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 6 Mar 2023 06:10:59 -0600 Subject: [PATCH 341/551] Add Manual host banning to PgCat (#340) Sometimes we want an admin to be able to ban a host for some time to route traffic away from that host for reasons like partial outages, replication lag, and scheduled maintenance. We can achieve this today using a configuration update but a quicker approach is to send a control command to PgCat that bans the replica for some specified duration. This command does not change the current banning rules like Primaries cannot be banned When all replicas are banned, all replicas are unbanned --- src/admin.rs | 172 +++++++++++++++++++++++++++++++++++++++ src/client.rs | 13 +-- src/errors.rs | 1 + src/pool.rs | 57 +++++++++++-- tests/ruby/admin_spec.rb | 70 ++++++++++++++++ 5 files changed, 300 insertions(+), 13 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 7f7ada565..c90f28ea2 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,9 +1,12 @@ +use crate::config::Role; +use crate::pool::BanReason; /// Admin database. use bytes::{Buf, BufMut, BytesMut}; use log::{error, info, trace}; use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; use std::collections::HashMap; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio::time::Instant; use crate::config::{get_config, reload_config, VERSION}; @@ -53,6 +56,14 @@ where let query_parts: Vec<&str> = query.trim_end_matches(';').split_whitespace().collect(); match query_parts[0].to_ascii_uppercase().as_str() { + "BAN" => { + trace!("BAN"); + ban(stream, query_parts).await + } + "UNBAN" => { + trace!("UNBAN"); + unban(stream, query_parts).await + } "RELOAD" => { trace!("RELOAD"); reload(stream, client_server_map).await @@ -74,6 +85,10 @@ where shutdown(stream).await } "SHOW" => match query_parts[1].to_ascii_uppercase().as_str() { + "BANS" => { + trace!("SHOW BANS"); + show_bans(stream).await + } "CONFIG" => { trace!("SHOW CONFIG"); show_config(stream).await @@ -350,6 +365,163 @@ where custom_protocol_response_ok(stream, "SET").await } +/// Bans a host from being used +async fn ban(stream: &mut T, tokens: Vec<&str>) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let host = match tokens.get(1) { + Some(host) => host, + None => return error_response(stream, "usage: BAN hostname duration_seconds").await, + }; + + let duration_seconds = match tokens.get(2) { + Some(duration_seconds) => match duration_seconds.parse::() { + Ok(duration_seconds) => duration_seconds, + Err(_) => { + return error_response(stream, "duration_seconds must be an integer").await; + } + }, + None => return error_response(stream, "usage: BAN hostname duration_seconds").await, + }; + + if duration_seconds <= 0 { + return error_response(stream, "duration_seconds must be >= 0").await; + } + + let columns = vec![ + ("db", DataType::Text), + ("user", DataType::Text), + ("role", DataType::Text), + ("host", DataType::Text), + ]; + let mut res = BytesMut::new(); + res.put(row_description(&columns)); + + for (id, pool) in get_all_pools().iter() { + for address in pool.get_addresses_from_host(host) { + if !pool.is_banned(&address) { + pool.ban(&address, BanReason::AdminBan(duration_seconds), -1); + res.put(data_row(&vec![ + id.db.clone(), + id.user.clone(), + address.role.to_string(), + address.host, + ])); + } + } + } + + res.put(command_complete("BAN")); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await +} + +/// Clear a host for use +async fn unban(stream: &mut T, tokens: Vec<&str>) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let host = match tokens.get(1) { + Some(host) => host, + None => return error_response(stream, "UNBAN command requires a hostname to unban").await, + }; + + let columns = vec![ + ("db", DataType::Text), + ("user", DataType::Text), + ("role", DataType::Text), + ("host", DataType::Text), + ]; + let mut res = BytesMut::new(); + res.put(row_description(&columns)); + + for (id, pool) in get_all_pools().iter() { + for address in pool.get_addresses_from_host(host) { + if pool.is_banned(&address) { + pool.unban(&address); + res.put(data_row(&vec![ + id.db.clone(), + id.user.clone(), + address.role.to_string(), + address.host, + ])); + } + } + } + + res.put(command_complete("UNBAN")); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await +} + +/// Shows all the bans +async fn show_bans(stream: &mut T) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let columns = vec![ + ("db", DataType::Text), + ("user", DataType::Text), + ("role", DataType::Text), + ("host", DataType::Text), + ("reason", DataType::Text), + ("ban_time", DataType::Text), + ("ban_duration_seconds", DataType::Text), + ("ban_remaining_seconds", DataType::Text), + ]; + let mut res = BytesMut::new(); + res.put(row_description(&columns)); + + // The block should be pretty quick so we cache the time outside + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs() as i64; + + for (id, pool) in get_all_pools().iter() { + for (address, (ban_reason, ban_time)) in pool.get_bans().iter() { + let ban_duration = match ban_reason { + BanReason::AdminBan(duration) => *duration, + _ => pool.settings.ban_time, + }; + let remaining = ban_duration - (now - ban_time.timestamp()); + if remaining <= 0 { + continue; + } + res.put(data_row(&vec![ + id.db.clone(), + id.user.clone(), + address.role.to_string(), + address.host.clone(), + format!("{:?}", ban_reason), + ban_time.to_string(), + ban_duration.to_string(), + remaining.to_string(), + ])); + } + } + + res.put(command_complete("SHOW BANS")); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await +} + /// Reload the configuration file without restarting the process. async fn reload(stream: &mut T, client_server_map: ClientServerMap) -> Result<(), Error> where diff --git a/src/client.rs b/src/client.rs index 11f4cc67e..2e04c2a92 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,6 +1,9 @@ +use crate::errors::Error; +use crate::pool::BanReason; /// Handle clients by pretending to be a PostgreSQL server. use bytes::{Buf, BufMut, BytesMut}; use log::{debug, error, info, trace, warn}; + use std::collections::HashMap; use std::time::Instant; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; @@ -11,7 +14,7 @@ use tokio::sync::mpsc::Sender; use crate::admin::{generate_server_info_for_admin, handle_admin}; use crate::config::{get_config, Address, PoolMode}; use crate::constants::*; -use crate::errors::Error; + use crate::messages::*; use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; use crate::query_router::{Command, QueryRouter}; @@ -1111,7 +1114,7 @@ where match server.send(message).await { Ok(_) => Ok(()), Err(err) => { - pool.ban(address, self.process_id); + pool.ban(address, BanReason::MessageSendFailed, self.process_id); Err(err) } } @@ -1133,7 +1136,7 @@ where Ok(result) => match result { Ok(message) => Ok(message), Err(err) => { - pool.ban(address, self.process_id); + pool.ban(address, BanReason::MessageReceiveFailed, self.process_id); error_response_terminal( &mut self.write, &format!("error receiving data from server: {:?}", err), @@ -1148,7 +1151,7 @@ where address, pool.settings.user.username ); server.mark_bad(); - pool.ban(address, self.process_id); + pool.ban(address, BanReason::StatementTimeout, self.process_id); error_response_terminal(&mut self.write, "pool statement timeout").await?; Err(Error::StatementTimeout) } @@ -1157,7 +1160,7 @@ where match server.recv().await { Ok(message) => Ok(message), Err(err) => { - pool.ban(address, self.process_id); + pool.ban(address, BanReason::MessageReceiveFailed, self.process_id); error_response_terminal( &mut self.write, &format!("error receiving data from server: {:?}", err), diff --git a/src/errors.rs b/src/errors.rs index 4ac23a855..310243c09 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -6,6 +6,7 @@ pub enum Error { SocketError(String), ClientBadStartup, ProtocolSyncError(String), + BadQuery(String), ServerError, BadConfig, AllServersDown, diff --git a/src/pool.rs b/src/pool.rs index cca95776f..0a0a53fab 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -29,7 +29,7 @@ pub type SecretKey = i32; pub type ServerHost = String; pub type ServerPort = u16; -pub type BanList = Arc>>>; +pub type BanList = Arc>>>; pub type ClientServerMap = Arc>>; pub type PoolMap = HashMap; @@ -38,6 +38,17 @@ pub type PoolMap = HashMap; /// The pool is recreated dynamically when the config is reloaded. pub static POOLS: Lazy> = Lazy::new(|| ArcSwap::from_pointee(HashMap::default())); +// Reasons for banning a server. +#[derive(Debug, PartialEq, Clone)] +pub enum BanReason { + FailedHealthCheck, + MessageSendFailed, + MessageReceiveFailed, + FailedCheckout, + StatementTimeout, + AdminBan(i64), +} + /// An identifier for a PgCat pool, /// a database visible to clients. #[derive(Hash, Debug, Clone, PartialEq, Eq)] @@ -489,7 +500,7 @@ impl ConnectionPool { Ok(conn) => conn, Err(err) => { error!("Banning instance {:?}, error: {:?}", address, err); - self.ban(address, client_process_id); + self.ban(address, BanReason::FailedCheckout, client_process_id); self.stats .client_checkout_error(client_process_id, address.id); continue; @@ -582,14 +593,14 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(&address, client_process_id); + self.ban(&address, BanReason::FailedHealthCheck, client_process_id); return false; } /// Ban an address (i.e. replica). It no longer will serve /// traffic for any new transactions. Existing transactions on that replica /// will finish successfully or error out to the clients. - pub fn ban(&self, address: &Address, client_id: i32) { + pub fn ban(&self, address: &Address, reason: BanReason, client_id: i32) { // Primary can never be banned if address.role == Role::Primary { return; @@ -599,12 +610,12 @@ impl ConnectionPool { let mut guard = self.banlist.write(); error!("Banning {:?}", address); self.stats.client_ban_error(client_id, address.id); - guard[address.shard].insert(address.clone(), now); + guard[address.shard].insert(address.clone(), (reason, now)); } /// Clear the replica to receive traffic again. Takes effect immediately /// for all new transactions. - pub fn _unban(&self, address: &Address) { + pub fn unban(&self, address: &Address) { let mut guard = self.banlist.write(); guard[address.shard].remove(address); } @@ -653,9 +664,14 @@ impl ConnectionPool { // Check if ban time is expired let read_guard = self.banlist.read(); let exceeded_ban_time = match read_guard[address.shard].get(address) { - Some(timestamp) => { + Some((ban_reason, timestamp)) => { let now = chrono::offset::Utc::now().naive_utc(); - now.timestamp() - timestamp.timestamp() > self.settings.ban_time + match ban_reason { + BanReason::AdminBan(duration) => { + now.timestamp() - timestamp.timestamp() > *duration + } + _ => now.timestamp() - timestamp.timestamp() > self.settings.ban_time, + } } None => return true, }; @@ -679,6 +695,31 @@ impl ConnectionPool { self.databases.len() } + pub fn get_bans(&self) -> Vec<(Address, (BanReason, NaiveDateTime))> { + let mut bans: Vec<(Address, (BanReason, NaiveDateTime))> = Vec::new(); + let guard = self.banlist.read(); + for banlist in guard.iter() { + for (address, (reason, timestamp)) in banlist.iter() { + bans.push((address.clone(), (reason.clone(), timestamp.clone()))); + } + } + return bans; + } + + /// Get the address from the host url + pub fn get_addresses_from_host(&self, host: &str) -> Vec
{ + let mut addresses = Vec::new(); + for shard in 0..self.shards() { + for server in 0..self.servers(shard) { + let address = self.address(shard, server); + if address.host == host { + addresses.push(address.clone()); + } + } + } + addresses + } + /// Get the number of servers (primary and replicas) /// configured for a shard. pub fn servers(&self, shard: usize) -> usize { diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index 783641567..f69c3df6f 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -287,6 +287,76 @@ end end + describe "Manual Banning" do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 10) } + before do + new_configs = processes.pgcat.current_config + # Prevent immediate unbanning when we ban localhost + new_configs["pools"]["sharded_db"]["shards"]["0"]["servers"][0][0] = "127.0.0.1" + new_configs["pools"]["sharded_db"]["shards"]["0"]["servers"][1][0] = "127.0.0.1" + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + end + + describe "BAN/UNBAN and SHOW BANS" do + it "bans/unbans hosts" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + + # Returns a list of the banned addresses + results = admin_conn.async_exec("BAN localhost 10").to_a + expect(results.count).to eq(2) + expect(results.map{ |r| r["host"] }.uniq).to eq(["localhost"]) + + # Subsequent calls should yield no results + results = admin_conn.async_exec("BAN localhost 10").to_a + expect(results.count).to eq(0) + + results = admin_conn.async_exec("SHOW BANS").to_a + expect(results.count).to eq(2) + expect(results.map{ |r| r["host"] }.uniq).to eq(["localhost"]) + + # Returns a list of the unbanned addresses + results = admin_conn.async_exec("UNBAN localhost").to_a + expect(results.count).to eq(2) + expect(results.map{ |r| r["host"] }.uniq).to eq(["localhost"]) + + # Subsequent calls should yield no results + results = admin_conn.async_exec("UNBAN localhost").to_a + expect(results.count).to eq(0) + + results = admin_conn.async_exec("SHOW BANS").to_a + expect(results.count).to eq(0) + end + + it "honors ban duration" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + + # Returns a list of the banned addresses + results = admin_conn.async_exec("BAN localhost 1").to_a + expect(results.count).to eq(2) + expect(results.map{ |r| r["host"] }.uniq).to eq(["localhost"]) + + sleep(2) + + # After 2 seconds the ban should be lifted + results = admin_conn.async_exec("SHOW BANS").to_a + expect(results.count).to eq(0) + end + + it "can handle bad input" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + + expect { admin_conn.async_exec("BAN").to_a }.to raise_error(PG::SystemError) + expect { admin_conn.async_exec("BAN a").to_a }.to raise_error(PG::SystemError) + expect { admin_conn.async_exec("BAN a a").to_a }.to raise_error(PG::SystemError) + expect { admin_conn.async_exec("BAN a -5").to_a }.to raise_error(PG::SystemError) + expect { admin_conn.async_exec("BAN a 0").to_a }.to raise_error(PG::SystemError) + expect { admin_conn.async_exec("BAN a a a").to_a }.to raise_error(PG::SystemError) + expect { admin_conn.async_exec("UNBAN").to_a }.to raise_error(PG::SystemError) + end + end + end + describe "SHOW users" do it "returns the right users" do admin_conn = PG::connect(processes.pgcat.admin_connection_string) From 352dd0b90e48d3f822988bbcdbf78ced932354c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 08:57:44 -0500 Subject: [PATCH 342/551] chore(deps): bump async-trait from 0.1.64 to 0.1.66 (#342) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.64 to 0.1.66. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/compare/0.1.64...0.1.66) --- updated-dependencies: - dependency-name: async-trait dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 34850206e..06c35e17c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,9 +28,9 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" dependencies = [ "proc-macro2", "quote", From 6846209ac3d57eb2815629806521d57e6020b594 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Mar 2023 01:18:10 -0800 Subject: [PATCH 343/551] chore(deps): bump sqlparser from 0.31.0 to 0.32.0 (#343) Bumps [sqlparser](https://github.com/sqlparser-rs/sqlparser-rs) from 0.31.0 to 0.32.0. - [Release notes](https://github.com/sqlparser-rs/sqlparser-rs/releases) - [Changelog](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/sqlparser-rs/sqlparser-rs/compare/v0.31.0...v0.32.0) --- updated-dependencies: - dependency-name: sqlparser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06c35e17c..b0beb5d39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1054,9 +1054,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f064eb7b163863163c29801910f763c6bfb563b8a8ca4c54193da4c1eea57547" +checksum = "0366f270dbabb5cc2e4c88427dc4c08bba144f81e32fbd459a013f26a4d16aa0" dependencies = [ "log", ] diff --git a/Cargo.toml b/Cargo.toml index 9ea45b3d8..4be3c7670 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = "0.31.0" +sqlparser = "0.32.0" log = "0.4" arc-swap = "1" env_logger = "0.10" From 2a10f8c57fa66d8d0978f14345058acf7aa592d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Mar 2023 22:50:45 -0800 Subject: [PATCH 344/551] chore(deps): bump serde from 1.0.152 to 1.0.154 (#348) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.152 to 1.0.154. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.152...v1.0.154) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0beb5d39..0bbe6f343 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -960,9 +960,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "8cdd151213925e7f1ab45a9bbfb129316bd00799784b174b7cc7bcd16961c49e" [[package]] name = "serde_derive" From bb61c80a62a012a11b8423fa5d244a2db766d071 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Mar 2023 22:50:53 -0800 Subject: [PATCH 345/551] chore(deps): bump serde_derive from 1.0.152 to 1.0.154 (#347) Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.152 to 1.0.154. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.152...v1.0.154) --- updated-dependencies: - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0bbe6f343..9856f6e09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -966,9 +966,9 @@ checksum = "8cdd151213925e7f1ab45a9bbfb129316bd00799784b174b7cc7bcd16961c49e" [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "4fc80d722935453bcafdc2c9a73cd6fac4dc1938f0346035d84bf99fa9e33217" dependencies = [ "proc-macro2", "quote", From de0f8df802a9f4178e8e93c742e026e3ae769a96 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 10 Mar 2023 06:23:51 -0600 Subject: [PATCH 346/551] PgCat Query Mirroring (#341) This is an implementation of Query mirroring in PgCat (outlined here #302) In configs, we match mirror hosts with the servers handling the traffic. A mirror host will receive the same protocol messages as the main server it was matched with. This is done by creating an async task for each mirror server, it communicates with the main server through two channels, one for the protocol messages and one for the exit signal. The mirror server sends the protocol packets to the underlying PostgreSQL server. We receive from the underlying PostgreSQL server as soon as the data is available and we immediately discard it. We use bb8 to manage the life cycle of the connection, not for pooling since each mirror server handler is more or less single-threaded. We don't have any connection pooling in the mirrors. Matching each mirror connection to an actual server connection guarantees that we will not have more connections to any of the mirrors than the parent pool would allow. --- .circleci/config.yml | 8 +- .circleci/run_tests.sh | 6 +- .gitignore | 2 +- dev/docker-compose.yaml | 8 +- src/admin.rs | 1 - src/config.rs | 21 +++- src/lib.rs | 1 + src/main.rs | 1 + src/mirrors.rs | 169 ++++++++++++++++++++++++++++ src/pool.rs | 29 ++++- src/query_router.rs | 1 + src/server.rs | 27 +++++ tests/docker/docker-compose.yml | 8 +- tests/ruby/helpers/pg_instance.rb | 12 ++ tests/ruby/helpers/pgcat_process.rb | 8 +- tests/ruby/load_balancing_spec.rb | 1 - tests/ruby/mirrors_spec.rb | 90 +++++++++++++++ 17 files changed, 370 insertions(+), 23 deletions(-) create mode 100644 src/mirrors.rs create mode 100644 tests/ruby/mirrors_spec.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index f7aa899b7..a43d4bd7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,28 +18,28 @@ jobs: RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cinstrument-coverage" RUSTDOCFLAGS: "-Cpanic=abort" - image: postgres:14 - command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements"] + command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] environment: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 - image: postgres:14 - command: ["postgres", "-p", "7432", "-c", "shared_preload_libraries=pg_stat_statements"] + command: ["postgres", "-p", "7432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] environment: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 - image: postgres:14 - command: ["postgres", "-p", "8432", "-c", "shared_preload_libraries=pg_stat_statements"] + command: ["postgres", "-p", "8432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] environment: POSTGRES_USER: postgres POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 - image: postgres:14 - command: ["postgres", "-p", "9432", "-c", "shared_preload_libraries=pg_stat_statements"] + command: ["postgres", "-p", "9432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] environment: POSTGRES_USER: postgres POSTGRES_DB: postgres diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 644f22e83..a5cfab0b1 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -92,12 +92,12 @@ sed -i 's/statement_timeout = 100/statement_timeout = 0/' .circleci/pgcat.toml kill -SIGHUP $(pgrep pgcat) # Reload config again # -# ActiveRecord tests +# Integration tests and ActiveRecord tests # cd tests/ruby sudo bundle install -bundle exec ruby tests.rb || exit 1 -bundle exec rspec *_spec.rb || exit 1 +bundle exec ruby tests.rb --format documentation || exit 1 +bundle exec rspec *_spec.rb --format documentation || exit 1 cd ../.. # diff --git a/.gitignore b/.gitignore index b3ca0139d..0b4361648 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ /target *.deb .vscode -.profraw +*.profraw cov/ lcov.info diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index ee609e0fe..da759383f 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -33,7 +33,7 @@ services: <<: *common-env-pg POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 PGPORT: 5432 - command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "5432"] + command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] pg2: <<: *common-definition-pg @@ -41,21 +41,21 @@ services: <<: *common-env-pg POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 PGPORT: 7432 - command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "7432"] + command: ["postgres", "-p", "7432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] pg3: <<: *common-definition-pg environment: <<: *common-env-pg POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 PGPORT: 8432 - command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "8432"] + command: ["postgres", "-p", "8432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] pg4: <<: *common-definition-pg environment: <<: *common-env-pg POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 PGPORT: 9432 - command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "9432"] + command: ["postgres", "-p", "9432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] toxiproxy: build: . diff --git a/src/admin.rs b/src/admin.rs index c90f28ea2..feea3a153 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,4 +1,3 @@ -use crate::config::Role; use crate::pool::BanReason; /// Admin database. use bytes::{Buf, BufMut, BytesMut}; diff --git a/src/config.rs b/src/config.rs index 517cabce1..9b90ebeae 100644 --- a/src/config.rs +++ b/src/config.rs @@ -29,6 +29,8 @@ pub enum Role { Primary, #[serde(alias = "replica", alias = "Replica")] Replica, + #[serde(alias = "mirror", alias = "Mirror")] + Mirror, } impl ToString for Role { @@ -36,6 +38,7 @@ impl ToString for Role { match *self { Role::Primary => "primary".to_string(), Role::Replica => "replica".to_string(), + Role::Mirror => "mirror".to_string(), } } } @@ -90,6 +93,9 @@ pub struct Address { /// The name of this pool (i.e. database name visible to the client). pub pool_name: String, + + /// List of addresses to receive mirrored traffic. + pub mirrors: Vec
, } impl Default for Address { @@ -105,6 +111,7 @@ impl Default for Address { role: Role::Replica, username: String::from("username"), pool_name: String::from("pool_name"), + mirrors: Vec::new(), } } } @@ -114,11 +121,14 @@ impl Address { pub fn name(&self) -> String { match self.role { Role::Primary => format!("{}_shard_{}_primary", self.pool_name, self.shard), - Role::Replica => format!( "{}_shard_{}_replica_{}", self.pool_name, self.shard, self.replica_number ), + Role::Mirror => format!( + "{}_shard_{}_mirror_{}", + self.pool_name, self.shard, self.replica_number + ), } } } @@ -465,11 +475,19 @@ pub struct ServerConfig { pub role: Role, } +#[derive(Clone, PartialEq, Serialize, Deserialize, Debug, Hash, Eq)] +pub struct MirrorServerConfig { + pub host: String, + pub port: u16, + pub mirroring_target_index: usize, +} + /// Shard configuration. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Hash, Eq)] pub struct Shard { pub database: String, pub servers: Vec, + pub mirrors: Option>, } impl Shard { @@ -518,6 +536,7 @@ impl Default for Shard { port: 5432, role: Role::Primary, }], + mirrors: None, database: String::from("postgres"), } } diff --git a/src/lib.rs b/src/lib.rs index 63eae59b7..67aa9cba0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ pub mod config; pub mod constants; pub mod errors; pub mod messages; +pub mod mirrors; pub mod multi_logger; pub mod pool; pub mod scram; diff --git a/src/main.rs b/src/main.rs index b3ef77c30..e2ff5d8d8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -66,6 +66,7 @@ mod config; mod constants; mod errors; mod messages; +mod mirrors; mod multi_logger; mod pool; mod prometheus; diff --git a/src/mirrors.rs b/src/mirrors.rs new file mode 100644 index 000000000..6a59172b7 --- /dev/null +++ b/src/mirrors.rs @@ -0,0 +1,169 @@ +/// A mirrored PostgreSQL client. +/// Packets arrive to us through a channel from the main client and we send them to the server. +use bb8::Pool; +use bytes::{Bytes, BytesMut}; + +use crate::config::{get_config, Address, Role, User}; +use crate::pool::{ClientServerMap, ServerPool}; +use crate::stats::get_reporter; +use log::{error, info, trace, warn}; +use tokio::sync::mpsc::{channel, Receiver, Sender}; + +pub struct MirroredClient { + address: Address, + user: User, + database: String, + bytes_rx: Receiver, + disconnect_rx: Receiver<()>, +} + +impl MirroredClient { + async fn create_pool(&self) -> Pool { + let config = get_config(); + let default = std::time::Duration::from_millis(10_000).as_millis() as u64; + let (connection_timeout, idle_timeout) = match config.pools.get(&self.address.pool_name) { + Some(cfg) => ( + cfg.connect_timeout.unwrap_or(default), + cfg.idle_timeout.unwrap_or(default), + ), + None => (default, default), + }; + + let manager = ServerPool::new( + self.address.clone(), + self.user.clone(), + self.database.as_str(), + ClientServerMap::default(), + get_reporter(), + ); + + Pool::builder() + .max_size(1) + .connection_timeout(std::time::Duration::from_millis(connection_timeout)) + .idle_timeout(Some(std::time::Duration::from_millis(idle_timeout))) + .test_on_check_out(false) + .build(manager) + .await + .unwrap() + } + + pub fn start(mut self) { + tokio::spawn(async move { + let pool = self.create_pool().await; + let address = self.address.clone(); + loop { + let mut server = match pool.get().await { + Ok(server) => server, + Err(err) => { + error!( + "Failed to get connection from pool, Discarding message {:?}, {:?}", + err, + address.clone() + ); + continue; + } + }; + + tokio::select! { + // Exit channel events + _ = self.disconnect_rx.recv() => { + info!("Got mirror exit signal, exiting {:?}", address.clone()); + break; + } + + // Incoming data from server (we read to clear the socket buffer and discard the data) + recv_result = server.recv() => { + match recv_result { + Ok(message) => trace!("Received from mirror: {} {:?}", String::from_utf8_lossy(&message[..]), address.clone()), + Err(err) => { + server.mark_bad(); + error!("Failed to receive from mirror {:?} {:?}", err, address.clone()); + } + } + } + + // Messages to send to the server + message = self.bytes_rx.recv() => { + match message { + Some(bytes) => { + match server.send(&BytesMut::from(&bytes[..])).await { + Ok(_) => trace!("Sent to mirror: {} {:?}", String::from_utf8_lossy(&bytes[..]), address.clone()), + Err(err) => { + server.mark_bad(); + error!("Failed to send to mirror, Discarding message {:?}, {:?}", err, address.clone()) + } + } + } + None => { + info!("Mirror channel closed, exiting {:?}", address.clone()); + break; + }, + } + } + } + } + }); + } +} +pub struct MirroringManager { + pub byte_senders: Vec>, + pub disconnect_senders: Vec>, +} +impl MirroringManager { + pub fn from_addresses( + user: User, + database: String, + addresses: Vec
, + ) -> MirroringManager { + let mut byte_senders: Vec> = vec![]; + let mut exit_senders: Vec> = vec![]; + + addresses.iter().for_each(|mirror| { + let (bytes_tx, bytes_rx) = channel::(500); + let (exit_tx, exit_rx) = channel::<()>(1); + let mut addr = mirror.clone(); + addr.role = Role::Mirror; + let client = MirroredClient { + user: user.clone(), + database: database.to_owned(), + address: addr, + bytes_rx, + disconnect_rx: exit_rx, + }; + exit_senders.push(exit_tx.clone()); + byte_senders.push(bytes_tx.clone()); + client.start(); + }); + + Self { + byte_senders: byte_senders, + disconnect_senders: exit_senders, + } + } + + pub fn send(self: &mut Self, bytes: &BytesMut) { + let cpy = bytes.clone().freeze(); + self.byte_senders + .iter_mut() + .for_each(|sender| match sender.try_send(cpy.clone()) { + Ok(_) => {} + Err(err) => { + warn!("Failed to send bytes to a mirror channel {}", err); + } + }); + } + + pub fn disconnect(self: &mut Self) { + self.disconnect_senders + .iter_mut() + .for_each(|sender| match sender.try_send(()) { + Ok(_) => {} + Err(err) => { + warn!( + "Failed to send disconnect signal to a mirror channel {}", + err + ); + } + }); + } +} diff --git a/src/pool.rs b/src/pool.rs index 0a0a53fab..3a6ec3e66 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -193,7 +193,7 @@ impl ConnectionPool { let config = get_config(); let mut new_pools = HashMap::new(); - let mut address_id = 0; + let mut address_id: usize = 0; for (pool_name, pool_config) in &config.pools { let new_pool_hash_value = pool_config.hash_value(); @@ -244,7 +244,33 @@ impl ConnectionPool { let mut servers = Vec::new(); let mut replica_number = 0; + // Load Mirror settings for (address_index, server) in shard.servers.iter().enumerate() { + let mut mirror_addresses = vec![]; + if let Some(mirror_settings_vec) = &shard.mirrors { + for (mirror_idx, mirror_settings) in + mirror_settings_vec.iter().enumerate() + { + if mirror_settings.mirroring_target_index != address_index { + continue; + } + mirror_addresses.push(Address { + id: address_id, + database: shard.database.clone(), + host: mirror_settings.host.clone(), + port: mirror_settings.port, + role: server.role, + address_index: mirror_idx, + replica_number, + shard: shard_idx.parse::().unwrap(), + username: user.username.clone(), + pool_name: pool_name.clone(), + mirrors: vec![], + }); + address_id += 1; + } + } + let address = Address { id: address_id, database: shard.database.clone(), @@ -256,6 +282,7 @@ impl ConnectionPool { shard: shard_idx.parse::().unwrap(), username: user.username.clone(), pool_name: pool_name.clone(), + mirrors: mirror_addresses, }; address_id += 1; diff --git a/src/query_router.rs b/src/query_router.rs index fff5bba8d..fbff68e94 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -223,6 +223,7 @@ impl QueryRouter { Command::ShowServerRole => match self.active_role { Some(Role::Primary) => Role::Primary.to_string(), Some(Role::Replica) => Role::Replica.to_string(), + Some(Role::Mirror) => Role::Mirror.to_string(), None => { if self.query_parser_enabled() { String::from("auto") diff --git a/src/server.rs b/src/server.rs index 1d9bcd14f..b3dbd6f7f 100644 --- a/src/server.rs +++ b/src/server.rs @@ -14,6 +14,7 @@ use crate::config::{Address, User}; use crate::constants::*; use crate::errors::Error; use crate::messages::*; +use crate::mirrors::MirroringManager; use crate::pool::ClientServerMap; use crate::scram::ScramSha256; use crate::stats::Reporter; @@ -68,6 +69,8 @@ pub struct Server { // Last time that a successful server send or response happened last_activity: SystemTime, + + mirror_manager: Option, } impl Server { @@ -334,6 +337,14 @@ impl Server { stats, application_name: String::new(), last_activity: SystemTime::now(), + mirror_manager: match address.mirrors.len() { + 0 => None, + _ => Some(MirroringManager::from_addresses( + user.clone(), + database.to_owned(), + address.mirrors.clone(), + )), + }, }; server.set_name("pgcat").await?; @@ -384,6 +395,7 @@ impl Server { /// Send messages to the server from the client. pub async fn send(&mut self, messages: &BytesMut) -> Result<(), Error> { + self.mirror_send(messages); self.stats.data_sent(messages.len(), self.server_id); match write_all_half(&mut self.write, messages).await { @@ -674,6 +686,20 @@ impl Server { pub fn mark_dirty(&mut self) { self.needs_cleanup = true; } + + pub fn mirror_send(&mut self, bytes: &BytesMut) { + match self.mirror_manager.as_mut() { + Some(manager) => manager.send(bytes), + None => (), + } + } + + pub fn mirror_disconnect(&mut self) { + match self.mirror_manager.as_mut() { + Some(manager) => manager.disconnect(), + None => (), + } + } } impl Drop for Server { @@ -681,6 +707,7 @@ impl Drop for Server { /// the socket is in non-blocking mode, so it may not be ready /// for a write. fn drop(&mut self) { + self.mirror_disconnect(); self.stats.server_disconnecting(self.server_id); let mut bytes = BytesMut::with_capacity(4); diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index e44dc5292..e57d85298 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -8,7 +8,7 @@ services: POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 - command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "5432"] + command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] pg2: image: postgres:14 network_mode: "service:main" @@ -17,7 +17,7 @@ services: POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 - command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "7432"] + command: ["postgres", "-p", "7432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] pg3: image: postgres:14 network_mode: "service:main" @@ -26,7 +26,7 @@ services: POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 - command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "8432"] + command: ["postgres", "-p", "8432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] pg4: image: postgres:14 network_mode: "service:main" @@ -35,7 +35,7 @@ services: POSTGRES_DB: postgres POSTGRES_PASSWORD: postgres POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 - command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "9432"] + command: ["postgres", "-p", "9432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] main: build: . command: ["bash", "/app/tests/docker/run.sh"] diff --git a/tests/ruby/helpers/pg_instance.rb b/tests/ruby/helpers/pg_instance.rb index 311645751..a38282486 100644 --- a/tests/ruby/helpers/pg_instance.rb +++ b/tests/ruby/helpers/pg_instance.rb @@ -38,6 +38,8 @@ def with_connection def reset reset_toxics reset_stats + drop_connections + sleep 0.1 end def toxiproxy @@ -66,12 +68,22 @@ def delete_proxy def reset_toxics Toxiproxy[@toxiproxy_name].toxics.each(&:destroy) + sleep 0.1 end def reset_stats with_connection { |c| c.async_exec("SELECT pg_stat_statements_reset()") } end + def drop_connections + username = with_connection { |c| c.async_exec("SELECT current_user")[0]["current_user"] } + with_connection { |c| c.async_exec("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND usename='#{username}'") } + end + + def count_connections + with_connection { |c| c.async_exec("SELECT COUNT(*) as count FROM pg_stat_activity")[0]["count"].to_i } + end + def count_query(query) with_connection { |c| c.async_exec("SELECT SUM(calls) FROM pg_stat_statements WHERE query = '#{query}'")[0]["sum"].to_i } end diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index 2108eafc2..6120c99f6 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -29,7 +29,7 @@ def initialize(log_level) else '../../target/debug/pgcat' end - + @command = "#{command_path} #{@config_filename}" FileUtils.cp("../../pgcat.toml", @config_filename) @@ -48,12 +48,14 @@ def update_config(config_hash) @original_config = current_config output_to_write = TOML::Generator.new(config_hash).body output_to_write = output_to_write.gsub(/,\s*["|'](\d+)["|']\s*,/, ',\1,') + output_to_write = output_to_write.gsub(/,\s*["|'](\d+)["|']\s*\]/, ',\1]') File.write(@config_filename, output_to_write) end def current_config - old_cfg = File.read(@config_filename) - loadable_string = old_cfg.gsub(/,\s*(\d+)\s*,/, ', "\1",') + loadable_string = File.read(@config_filename) + loadable_string = loadable_string.gsub(/,\s*(\d+)\s*,/, ', "\1",') + loadable_string = loadable_string.gsub(/,\s*(\d+)\s*\]/, ', "\1"]') TOML.load(loadable_string) end diff --git a/tests/ruby/load_balancing_spec.rb b/tests/ruby/load_balancing_spec.rb index e7b89ee8d..cd647406c 100644 --- a/tests/ruby/load_balancing_spec.rb +++ b/tests/ruby/load_balancing_spec.rb @@ -46,7 +46,6 @@ end end - expect(failed_count).to be <= 2 processes.all_databases.each do |instance| queries_routed = instance.count_select_1_plus_2 if processes.replicas[0..1].include?(instance) diff --git a/tests/ruby/mirrors_spec.rb b/tests/ruby/mirrors_spec.rb new file mode 100644 index 000000000..801df28cb --- /dev/null +++ b/tests/ruby/mirrors_spec.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true +require 'uri' +require_relative 'spec_helper' + +describe "Query Mirroing" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 10) } + let(:mirror_pg) { PgInstance.new(8432, "sharding_user", "sharding_user", "shard2")} + let(:pgcat_conn_str) { processes.pgcat.connection_string("sharded_db", "sharding_user") } + let(:mirror_host) { "localhost" } + + before do + new_configs = processes.pgcat.current_config + new_configs["pools"]["sharded_db"]["shards"]["0"]["mirrors"] = [ + [mirror_host, mirror_pg.port.to_s, "0"], + [mirror_host, mirror_pg.port.to_s, "0"], + [mirror_host, mirror_pg.port.to_s, "0"], + ] + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + end + + after do + processes.all_databases.map(&:reset) + mirror_pg.reset + processes.pgcat.shutdown + end + + it "can mirror a query" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + runs = 15 + runs.times { conn.async_exec("SELECT 1 + 2") } + sleep 0.5 + expect(processes.all_databases.first.count_select_1_plus_2).to eq(runs) + expect(mirror_pg.count_select_1_plus_2).to eq(runs * 3) + end + + context "when main server connection is closed" do + it "closes the mirror connection" do + baseline_count = processes.all_databases.first.count_connections + 5.times do |i| + # Force pool cycling to detect zombie mirror connections + new_configs = processes.pgcat.current_config + new_configs["pools"]["sharded_db"]["idle_timeout"] = 5000 + i + new_configs["pools"]["sharded_db"]["shards"]["0"]["mirrors"] = [ + [mirror_host, mirror_pg.port.to_s, "0"], + [mirror_host, mirror_pg.port.to_s, "0"], + [mirror_host, mirror_pg.port.to_s, "0"], + ] + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + end + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SELECT 1 + 2") + sleep 0.5 + # Expect same number of connection even after pool cycling + expect(processes.all_databases.first.count_connections).to be < baseline_count + 2 + end + end + + xcontext "when mirror server goes down temporarily" do + it "continues to transmit queries after recovery" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + mirror_pg.take_down do + conn.async_exec("SELECT 1 + 2") + sleep 0.1 + end + 10.times { conn.async_exec("SELECT 1 + 2") } + sleep 1 + expect(mirror_pg.count_select_1_plus_2).to be >= 2 + end + end + + context "when a mirror is down" do + let(:mirror_host) { "badhost" } + + it "does not fail to send the main query" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + # No Errors here + conn.async_exec("SELECT 1 + 2") + expect(processes.all_databases.first.count_select_1_plus_2).to eq(1) + end + + it "does not fail to send the main query (even after thousands of mirror attempts)" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + # No Errors here + 1000.times { conn.async_exec("SELECT 1 + 2") } + expect(processes.all_databases.first.count_select_1_plus_2).to eq(1000) + end + end +end From da1dae04c05065a5994f1c9e71328319dc74f983 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 10 Mar 2023 07:39:42 -0600 Subject: [PATCH 347/551] Reorder fields in Shard to avoid ValueAfterTable errors (#349) --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 9b90ebeae..5adf8d5e8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -486,8 +486,8 @@ pub struct MirrorServerConfig { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Hash, Eq)] pub struct Shard { pub database: String, - pub servers: Vec, pub mirrors: Option>, + pub servers: Vec, } impl Shard { From fa8ab770a4bc0b39eec05d979e2d2da2210a05a0 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 10 Mar 2023 07:55:22 -0800 Subject: [PATCH 348/551] Extended query protocol sharding (#339) * Prepared stmt sharding s tests * len check * remove python test * latest rust * move that to debug for sure * Add the actual tests * latest image * Update tests/ruby/sharding_spec.rb --- .dockerignore | 2 + Dockerfile.ci | 4 +- dev/script/console | 8 +- src/client.rs | 38 ++++- src/query_router.rs | 241 +++++++++++++++++++++++++++-- tests/ruby/helpers/pgcat_helper.rb | 5 +- tests/ruby/sharding_spec.rb | 51 ++++++ 7 files changed, 320 insertions(+), 29 deletions(-) create mode 100644 tests/ruby/sharding_spec.rb diff --git a/.dockerignore b/.dockerignore index ff42aaaaf..1063ad186 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,5 @@ target/ tests/ tracing/ .circleci/ +.git/ +dev/ diff --git a/Dockerfile.ci b/Dockerfile.ci index 42d213a8c..4503e8703 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,7 +1,7 @@ -FROM cimg/rust:1.62.0 +FROM cimg/rust:1.67.1 RUN sudo apt-get update && \ sudo apt-get install -y \ - psmisc postgresql-contrib-12 postgresql-client-12 libpq-dev \ + psmisc postgresql-contrib-14 postgresql-client-14 libpq-dev \ ruby ruby-dev python3 python3-pip \ lcov llvm-11 iproute2 && \ sudo apt-get upgrade curl && \ diff --git a/dev/script/console b/dev/script/console index f2a12d65b..27715d1bd 100755 --- a/dev/script/console +++ b/dev/script/console @@ -3,4 +3,10 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" export HOST_UID="$(id -u)" export HOST_GID="$(id -g)" -docker-compose -f "${DIR}/../docker-compose.yaml" run --rm pgcat-shell + +if [[ "${1}" == "down" ]]; then + docker-compose -f "${DIR}/../docker-compose.yaml" down + exit 0 +else + docker-compose -f "${DIR}/../docker-compose.yaml" run --rm pgcat-shell +fi diff --git a/src/client.rs b/src/client.rs index 2e04c2a92..bb70a6567 100644 --- a/src/client.rs +++ b/src/client.rs @@ -675,14 +675,42 @@ where // allocate a connection, we wouldn't be able to send back an error message // to the client so we buffer them and defer the decision to error out or not // to when we get the S message - 'P' | 'B' | 'D' | 'E' => { + 'D' | 'E' => { self.buffer.put(&message[..]); continue; } + + 'Q' => { + if query_router.query_parser_enabled() { + query_router.infer(&message); + } + } + + 'P' => { + self.buffer.put(&message[..]); + + if query_router.query_parser_enabled() { + query_router.infer(&message); + } + + continue; + } + + 'B' => { + self.buffer.put(&message[..]); + + if query_router.query_parser_enabled() { + query_router.infer_shard_from_bind(&message); + } + + continue; + } + 'X' => { debug!("Client disconnecting"); return Ok(()); } + _ => (), } @@ -711,11 +739,7 @@ where // Handle all custom protocol commands, if any. match query_router.try_execute_command(&message) { // Normal query, not a custom command. - None => { - if query_router.query_parser_enabled() { - query_router.infer(&message); - } - } + None => (), // SET SHARD TO Some((Command::SetShard, _)) => { @@ -727,7 +751,7 @@ where error_response( &mut self.write, &format!( - "shard {} is more than configured {}, staying on shard {}", + "shard {} is more than configured {}, staying on shard {} (shard numbers start at 0)", query_router.shard(), pool.shards(), current_shard, diff --git a/src/query_router.rs b/src/query_router.rs index fbff68e94..578c73901 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -43,6 +43,20 @@ pub enum Command { ShowPrimaryReads, } +#[derive(PartialEq, Debug)] +pub enum ShardingKey { + Value(i64), + Placeholder(i16), +} + +#[derive(Clone, Debug)] +enum ParameterFormat { + Text, + Binary, + Uniform(Box), + Specified(Vec), +} + /// Quickly test for match when a query is received. static CUSTOM_SQL_REGEX_SET: OnceCell = OnceCell::new(); @@ -65,6 +79,9 @@ pub struct QueryRouter { /// Pool configuration. pool_settings: PoolSettings, + + // Placeholders from prepared statement. + placeholders: Vec, } impl QueryRouter { @@ -103,6 +120,7 @@ impl QueryRouter { query_parser_enabled: None, primary_reads_enabled: None, pool_settings: PoolSettings::default(), + placeholders: Vec::new(), } } @@ -307,10 +325,10 @@ impl QueryRouter { } /// Try to infer which server to connect to based on the contents of the query. - pub fn infer(&mut self, message_buffer: &BytesMut) -> bool { + pub fn infer(&mut self, message: &BytesMut) -> bool { debug!("Inferring role"); - let mut message_cursor = Cursor::new(message_buffer); + let mut message_cursor = Cursor::new(message); let code = message_cursor.get_u8() as char; let _len = message_cursor.get_i32() as usize; @@ -332,8 +350,7 @@ impl QueryRouter { let query = message_cursor.read_string().unwrap(); debug!("Prepared statement: '{}'", query); - - query.replace('$', "") // Remove placeholders turning them into "values" + query } _ => return false, @@ -343,7 +360,7 @@ impl QueryRouter { Ok(ast) => ast, Err(err) => { // SELECT ... FOR UPDATE won't get parsed correctly. - error!("{}: {}", err, query); + debug!("{}: {}", err, query); self.active_role = Some(Role::Primary); return false; } @@ -404,9 +421,147 @@ impl QueryRouter { true } + /// Parse the shard number from the Bind message + /// which contains the arguments for a prepared statement. + /// + /// N.B.: Only supports anonymous prepared statements since we don't + /// keep a cache of them in PgCat. + pub fn infer_shard_from_bind(&mut self, message: &BytesMut) -> bool { + debug!("Parsing bind message"); + + let mut message_cursor = Cursor::new(message); + + let code = message_cursor.get_u8() as char; + let len = message_cursor.get_i32(); + + if code != 'B' { + debug!("Not a bind packet"); + return false; + } + + // Check message length + if message.len() != len as usize + 1 { + debug!( + "Message has wrong length, expected {}, but have {}", + len, + message.len() + ); + return false; + } + + // There are no shard keys in the prepared statement. + if self.placeholders.is_empty() { + debug!("There are no placeholders in the prepared statement that matched the automatic sharding key"); + return false; + } + + let sharder = Sharder::new( + self.pool_settings.shards, + self.pool_settings.sharding_function, + ); + + let mut shards = BTreeSet::new(); + + let _portal = message_cursor.read_string(); + let _name = message_cursor.read_string(); + + let num_params = message_cursor.get_i16(); + let parameter_format = match num_params { + 0 => ParameterFormat::Text, // Text + 1 => { + let param_format = message_cursor.get_i16(); + ParameterFormat::Uniform(match param_format { + 0 => Box::new(ParameterFormat::Text), + 1 => Box::new(ParameterFormat::Binary), + _ => unreachable!(), + }) + } + n => { + let mut v = Vec::with_capacity(n as usize); + for _ in 0..n { + let param_format = message_cursor.get_i16(); + v.push(match param_format { + 0 => ParameterFormat::Text, + 1 => ParameterFormat::Binary, + _ => unreachable!(), + }); + } + ParameterFormat::Specified(v) + } + }; + + let num_parameters = message_cursor.get_i16(); + + for i in 0..num_parameters { + let mut len = message_cursor.get_i32() as usize; + let format = match ¶meter_format { + ParameterFormat::Text => ParameterFormat::Text, + ParameterFormat::Uniform(format) => *format.clone(), + ParameterFormat::Specified(formats) => formats[i as usize].clone(), + _ => unreachable!(), + }; + + debug!("Parameter {} (len: {}): {:?}", i, len, format); + + // Postgres counts placeholders starting at 1 + let placeholder = i + 1; + + if self.placeholders.contains(&placeholder) { + let value = match format { + ParameterFormat::Text => { + let mut value = String::new(); + while len > 0 { + value.push(message_cursor.get_u8() as char); + len -= 1; + } + + match value.parse::() { + Ok(value) => value, + Err(_) => { + debug!("Error parsing bind value: {}", value); + continue; + } + } + } + + ParameterFormat::Binary => match len { + 2 => message_cursor.get_i16() as i64, + 4 => message_cursor.get_i32() as i64, + 8 => message_cursor.get_i64(), + _ => { + error!( + "Got wrong length for integer type parameter in bind: {}", + len + ); + continue; + } + }, + + _ => unreachable!(), + }; + + shards.insert(sharder.shard(value)); + } + } + + self.placeholders.clear(); + self.placeholders.shrink_to_fit(); + + // We only support querying one shard at a time. + // TODO: Support multi-shard queries some day. + if shards.len() == 1 { + debug!("Found one sharding key"); + self.set_shard(*shards.first().unwrap()); + true + } else { + debug!("Found no sharding keys"); + false + } + } + /// A `selection` is the `WHERE` clause. This parses /// the clause and extracts the sharding key, if present. - fn selection_parser(&self, expr: &Expr, table_names: &Vec>) -> Vec { + fn selection_parser(&self, expr: &Expr, table_names: &Vec>) -> Vec { let mut result = Vec::new(); let mut found = false; @@ -487,13 +642,25 @@ impl QueryRouter { Expr::Value(Value::Number(value, ..)) => { if found { match value.parse::() { - Ok(value) => result.push(value), + Ok(value) => result.push(ShardingKey::Value(value)), Err(_) => { debug!("Sharding key was not an integer: {}", value); } }; } } + + Expr::Value(Value::Placeholder(placeholder)) => { + match placeholder.replace("$", "").parse::() { + Ok(placeholder) => result.push(ShardingKey::Placeholder(placeholder)), + Err(_) => { + debug!( + "Prepared statement didn't have integer placeholders: {}", + placeholder + ); + } + } + } _ => (), }; } @@ -504,7 +671,7 @@ impl QueryRouter { } /// Try to figure out which shard the query should go to. - fn infer_shard(&self, query: &sqlparser::ast::Query) -> Option { + fn infer_shard(&mut self, query: &sqlparser::ast::Query) -> Option { let mut shards = BTreeSet::new(); let mut exprs = Vec::new(); @@ -569,6 +736,11 @@ impl QueryRouter { None => (), }; + let sharder = Sharder::new( + self.pool_settings.shards, + self.pool_settings.sharding_function, + ); + // Look for sharding keys in either the join condition // or the selection. for expr in exprs.iter() { @@ -577,14 +749,17 @@ impl QueryRouter { // TODO: Add support for prepared statements here. // This should just give us the position of the value in the `B` message. - let sharder = Sharder::new( - self.pool_settings.shards, - self.pool_settings.sharding_function, - ); - for value in sharding_keys { - let shard = sharder.shard(value); - shards.insert(shard); + match value { + ShardingKey::Value(value) => { + let shard = sharder.shard(value); + shards.insert(shard); + } + + ShardingKey::Placeholder(position) => { + self.placeholders.push(position); + } + }; } } } @@ -634,10 +809,14 @@ impl QueryRouter { /// Should we attempt to parse queries? pub fn query_parser_enabled(&self) -> bool { - match self.query_parser_enabled { + let enabled = match self.query_parser_enabled { None => self.pool_settings.query_parser_enabled, Some(value) => value, - } + }; + + debug!("Query parser enabled: {}", enabled); + + enabled } pub fn primary_reads_enabled(&self) -> bool { @@ -1066,4 +1245,32 @@ mod test { assert!(qr.infer(&simple_query("SELECT * FROM table_y WHERE another_key = 5"))); assert_eq!(qr.shard(), 0); } + + #[test] + fn test_prepared_statements() { + let stmt = "SELECT * FROM data WHERE id = $1"; + + let mut bind = BytesMut::from(&b"B"[..]); + + let mut payload = BytesMut::from(&b"\0\0"[..]); + payload.put_i16(0); + payload.put_i16(1); + payload.put_i32(1); + payload.put(&b"5"[..]); + payload.put_i16(0); + + bind.put_i32(payload.len() as i32 + 4); + bind.put(payload); + + let mut qr = QueryRouter::new(); + qr.pool_settings.automatic_sharding_key = Some("data.id".to_string()); + qr.pool_settings.shards = 3; + + assert!(qr.infer(&simple_query(stmt))); + assert_eq!(qr.placeholders.len(), 1); + + assert!(qr.infer_shard_from_bind(&bind)); + assert_eq!(qr.shard(), 2); + assert!(qr.placeholders.is_empty()); + } } diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index 544c8278b..c4ebab7f4 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -24,8 +24,9 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mod "default_role" => "any", "pool_mode" => pool_mode, "load_balancing_mode" => lb_mode, - "primary_reads_enabled" => false, - "query_parser_enabled" => false, + "primary_reads_enabled" => true, + "query_parser_enabled" => true, + "automatic_sharding_key" => "data.id", "sharding_function" => "pg_bigint_hash", "shards" => { "0" => { "database" => "shard0", "servers" => [["localhost", primary0.port.to_s, "primary"]] }, diff --git a/tests/ruby/sharding_spec.rb b/tests/ruby/sharding_spec.rb new file mode 100644 index 000000000..4c4053de8 --- /dev/null +++ b/tests/ruby/sharding_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true +require_relative 'spec_helper' + + +describe "Sharding" do + let(:processes) { Helpers::Pgcat.three_shard_setup("sharded_db", 5) } + + before do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + + # Setup the sharding data + 3.times do |i| + conn.exec("SET SHARD TO '#{i}'") + conn.exec("DELETE FROM data WHERE id > 0") + end + + 18.times do |i| + i = i + 1 + conn.exec("SET SHARDING KEY TO '#{i}'") + conn.exec("INSERT INTO data (id, value) VALUES (#{i}, 'value_#{i}')") + end + end + + after do + + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + + describe "automatic routing of extended procotol" do + it "can do it" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.exec("SET SERVER ROLE TO 'auto'") + + 18.times do |i| + result = conn.exec_params("SELECT * FROM data WHERE id = $1", [i + 1]) + expect(result.ntuples).to eq(1) + end + end + + it "can do it with multiple parameters" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.exec("SET SERVER ROLE TO 'auto'") + + 18.times do |i| + result = conn.exec_params("SELECT * FROM data WHERE id = $1 AND id = $2", [i + 1, i + 1]) + expect(result.ntuples).to eq(1) + end + end + end +end From 35408e08fa1aeaa1483b5ec26a1c1705aa53a5ac Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 10 Mar 2023 09:42:52 -0800 Subject: [PATCH 349/551] Build on 1.67 (#350) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a43d4bd7c..5e2d114a2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - - image: ghcr.io/levkk/pgcat-ci:latest + - image: ghcr.io/levkk/pgcat-ci:1.67 environment: RUST_LOG: info LLVM_PROFILE_FILE: /tmp/pgcat-%m-%p.profraw From 8cdab1909ede27ddabc9de1ee493eb56de1f802c Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 10 Mar 2023 22:00:28 -0600 Subject: [PATCH 350/551] Improve Config Documentation (#351) This PR adds a utility script that generates config documentation from pgcat.toml. Ideally, we'd want to generate the configs directly from config.rs where the actual defaults are set but this is a good start as we already had several undocumented config flags. --- CONFIG.md | 340 ++++++++++++++++++++++++++++++ README.md | 30 +-- pgcat.toml | 100 +++++---- utilities/generate_config_docs.py | 92 ++++++++ 4 files changed, 490 insertions(+), 72 deletions(-) create mode 100644 CONFIG.md create mode 100644 utilities/generate_config_docs.py diff --git a/CONFIG.md b/CONFIG.md new file mode 100644 index 000000000..b6dcc4780 --- /dev/null +++ b/CONFIG.md @@ -0,0 +1,340 @@ +# PgCat Configurations +## `general` Section + +### host +``` +path: general.host +default: "0.0.0.0" +``` + +What IP to run on, 0.0.0.0 means accessible from everywhere. + +### port +``` +path: general.port +default: 6432 +``` + +Port to run on, same as PgBouncer used in this example. + +### enable_prometheus_exporter +``` +path: general.enable_prometheus_exporter +default: true +``` + +Whether to enable prometheus exporter or not. + +### prometheus_exporter_port +``` +path: general.prometheus_exporter_port +default: 9930 +``` + +Port at which prometheus exporter listens on. + +### connect_timeout +``` +path: general.connect_timeout +default: 5000 # milliseconds +``` + +How long to wait before aborting a server connection (ms). + +### idle_timeout +``` +path: general.idle_timeout +default: 30000 # milliseconds +``` + +How long an idle connection with a server is left open (ms). + +### healthcheck_timeout +``` +path: general.healthcheck_timeout +default: 1000 # milliseconds +``` + +How much time to give the health check query to return with a result (ms). + +### healthcheck_delay +``` +path: general.healthcheck_delay +default: 30000 # milliseconds +``` + +How long to keep connection available for immediate re-use, without running a healthcheck query on it + +### shutdown_timeout +``` +path: general.shutdown_timeout +default: 60000 # milliseconds +``` + +How much time to give clients during shutdown before forcibly killing client connections (ms). + +### ban_time +``` +path: general.ban_time +default: 60 # seconds +``` + +How long to ban a server if it fails a health check (seconds). + +### log_client_connections +``` +path: general.log_client_connections +default: false +``` + +If we should log client connections + +### log_client_disconnections +``` +path: general.log_client_disconnections +default: false +``` + +If we should log client disconnections + +### autoreload +``` +path: general.autoreload +default: false +``` + +When set to true, PgCat reloads configs if it detects a change in the config file. + +### worker_threads +``` +path: general.worker_threads +default: 5 +``` + +Number of worker threads the Runtime will use (4 by default). + +### tcp_keepalives_idle +``` +path: general.tcp_keepalives_idle +default: 5 +``` + +Number of seconds of connection idleness to wait before sending a keepalive packet to the server. + +### tcp_keepalives_count +``` +path: general.tcp_keepalives_count +default: 5 +``` + +Number of unacknowledged keepalive packets allowed before giving up and closing the connection. + +### tcp_keepalives_interval +``` +path: general.tcp_keepalives_interval +default: 5 +``` + +Number of seconds between keepalive packets. + +### tls_certificate +``` +path: general.tls_certificate +default: +example: "server.cert" +``` + +Path to TLS Certficate file to use for TLS connections + +### tls_private_key +``` +path: general.tls_private_key +default: +example: "server.key" +``` + +Path to TLS private key file to use for TLS connections + +### admin_username +``` +path: general.admin_username +default: "admin_user" +``` + +User name to access the virtual administrative database (pgbouncer or pgcat) +Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. + +### admin_password +``` +path: general.admin_password +default: "admin_pass" +``` + +Password to access the virtual administrative database + +## `pools.` Section + +### pool_mode +``` +path: pools..pool_mode +default: "transaction" +``` + +Pool mode (see PgBouncer docs for more). +`session` one server connection per connected client +`transaction` one server connection per client transaction + +### load_balancing_mode +``` +path: pools..load_balancing_mode +default: "random" +``` + +Load balancing mode +`random` selects the server at random +`loc` selects the server with the least outstanding busy conncetions + +### default_role +``` +path: pools..default_role +default: "any" +``` + +If the client doesn't specify, PgCat routes traffic to this role by default. +`any` round-robin between primary and replicas, +`replica` round-robin between replicas only without touching the primary, +`primary` all queries go to the primary unless otherwise specified. + +### query_parser_enabled +``` +path: pools..query_parser_enabled +default: true +``` + +If Query Parser is enabled, we'll attempt to parse +every incoming query to determine if it's a read or a write. +If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, +we'll direct it to the primary. + +### primary_reads_enabled +``` +path: pools..primary_reads_enabled +default: true +``` + +If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for +load balancing of read queries. Otherwise, the primary will only be used for write +queries. The primary can always be explicitly selected with our custom protocol. + +### sharding_key_regex +``` +path: pools..sharding_key_regex +default: +example: '/\* sharding_key: (\d+) \*/' +``` + +Allow sharding commands to be passed as statement comments instead of +separate commands. If these are unset this functionality is disabled. + +### sharding_function +``` +path: pools..sharding_function +default: "pg_bigint_hash" +``` + +So what if you wanted to implement a different hashing function, +or you've already built one and you want this pooler to use it? +Current options: +`pg_bigint_hash`: PARTITION BY HASH (Postgres hashing function) +`sha1`: A hashing function based on SHA1 + +### automatic_sharding_key +``` +path: pools..automatic_sharding_key +default: +example: "data.id" +``` + +Automatically parse this from queries and route queries to the right shard! + +### idle_timeout +``` +path: pools..idle_timeout +default: 40000 +``` + +Idle timeout can be overwritten in the pool + +### connect_timeout +``` +path: pools..connect_timeout +default: 3000 +``` + +Connect timeout can be overwritten in the pool + +## `pools..users.` Section + +### username +``` +path: pools..users..username +default: "sharding_user" +``` + +Postgresql username + +### password +``` +path: pools..users..password +default: "sharding_user" +``` + +Postgresql password + +### pool_size +``` +path: pools..users..pool_size +default: 9 +``` + +Maximum number of server connections that can be established for this user +The maximum number of connection from a single Pgcat process to any database in the cluster +is the sum of pool_size across all users. + +### statement_timeout +``` +path: pools..users..statement_timeout +default: 0 +``` + +Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. +0 means it is disabled. + +## `pools..shards.` Section + +### servers +``` +path: pools..shards..servers +default: [["127.0.0.1", 5432, "primary"], ["localhost", 5432, "replica"]] +``` + +Array of servers in the shard, each server entry is an array of `[host, port, role]` + +### mirrors +``` +path: pools..shards..mirrors +default: +example: [["1.2.3.4", 5432, 0], ["1.2.3.4", 5432, 1]] +``` + +Array of mirrors for the shard, each mirror entry is an array of `[host, port, index of server in servers array]` +Traffic hitting the server identified by the index will be sent to the mirror. + +### database +``` +path: pools..shards..database +default: "shard0" +``` + +Database name (e.g. "postgres") + diff --git a/README.md b/README.md index a8723b603..ddbb661d4 100644 --- a/README.md +++ b/README.md @@ -39,35 +39,7 @@ PGPASSWORD=postgres psql -h 127.0.0.1 -p 6432 -U postgres -c 'SELECT 1' ### Config -| **Name** | **Description** | **Examples** | -|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------| -| **`general`** | | | -| `host` | The pooler will run on this host, 0.0.0.0 means accessible from everywhere. | `0.0.0.0` | -| `port` | The pooler will run on this port. | `6432` | -| `enable_prometheus_exporter` | Enable prometheus exporter which will export metrics in prometheus exposition format. | `true` | -| `prometheus_exporter_port` | Port at which prometheus exporter listens on. | `9930` | -| `pool_size` | Maximum allowed server connections per pool. Pools are separated for each user/shard/server role. The connections are allocated as needed. | `15` | -| `pool_mode` | The pool mode to use, i.e. `session` or `transaction`. | `transaction` | -| `connect_timeout` | Maximum time to establish a connection to a server (milliseconds). If reached, the server is banned and the next target is attempted. | `5000` | -| `healthcheck_timeout` | Maximum time to pass a health check (`SELECT 1`, milliseconds). If reached, the server is banned and the next target is attempted. | `1000` | -| `shutdown_timeout` | Maximum time to give clients during shutdown before forcibly killing client connections (ms). | `60000` | -| `healthcheck_delay` | How long to keep connection available for immediate re-use, without running a healthcheck query on it | `30000` | -| `ban_time` | Ban time for a server (seconds). It won't be allowed to serve transactions until the ban expires; failover targets will be used instead. | `60` | -| `autoreload` | Enable auto-reload of config after fixed time-interval. | `false` | -| | | | -| **`user`** | | | -| `name` | The user name. | `sharding_user` | -| `password` | The user password in plaintext. | `hunter2` | -| `statement_timeout` | Timeout in milliseconds for how long a query takes to execute | `0 (disabled)` -| | | | -| **`shards`** | Shards are numerically numbered starting from 0; the order in the config is preserved by the pooler to route queries accordingly. | `[shards.0]` | -| `servers` | List of servers to connect to and their roles. A server is: `[host, port, role]`, where `role` is either `primary` or `replica`. | `["127.0.0.1", 5432, "primary"]` | -| `database` | The name of the database to connect to. This is the same on all servers that are part of one shard. | | -| | | | -| **`query_router`** | | | -| `default_role` | Traffic is routed to this role by default (random), unless the client specifies otherwise. Default is `any`, for any role available. | `any`, `primary`, `replica` | -| `query_parser_enabled` | Enable the query parser which will inspect incoming queries and route them to a primary or replicas. | `false` | -| `primary_reads_enabled` | Enable this to allow read queries on the primary; otherwise read queries are routed to the replicas. | `true` | +(See Configurations page)[https://github.com/levkk/pgcat/blob/main/CONFIG.md] ## Local development diff --git a/pgcat.toml b/pgcat.toml index b5328b643..7903cdfb5 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -18,21 +18,21 @@ enable_prometheus_exporter = true prometheus_exporter_port = 9930 # How long to wait before aborting a server connection (ms). -connect_timeout = 5000 +connect_timeout = 5000 # milliseconds # How long an idle connection with a server is left open (ms). -idle_timeout = 30000 +idle_timeout = 30000 # milliseconds # How much time to give the health check query to return with a result (ms). -healthcheck_timeout = 1000 +healthcheck_timeout = 1000 # milliseconds # How long to keep connection available for immediate re-use, without running a healthcheck query on it -healthcheck_delay = 30000 +healthcheck_delay = 30000 # milliseconds # How much time to give clients during shutdown before forcibly killing client connections (ms). -shutdown_timeout = 60000 +shutdown_timeout = 60000 # milliseconds -# For how long to ban a server if it fails a health check (seconds). +# How long to ban a server if it fails a health check (seconds). ban_time = 60 # seconds # If we should log client connections @@ -41,40 +41,52 @@ log_client_connections = false # If we should log client disconnections log_client_disconnections = false -# Reload config automatically if it changes. +# When set to true, PgCat reloads configs if it detects a change in the config file. autoreload = false # Number of worker threads the Runtime will use (4 by default). worker_threads = 5 -# TLS +# Number of seconds of connection idleness to wait before sending a keepalive packet to the server. +tcp_keepalives_idle = 5 +# Number of unacknowledged keepalive packets allowed before giving up and closing the connection. +tcp_keepalives_count = 5 +# Number of seconds between keepalive packets. +tcp_keepalives_interval = 5 + +# Path to TLS Certficate file to use for TLS connections # tls_certificate = "server.cert" +# Path to TLS private key file to use for TLS connections # tls_private_key = "server.key" -# Credentials to access the virtual administrative database (pgbouncer or pgcat) +# User name to access the virtual administrative database (pgbouncer or pgcat) # Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. admin_username = "admin_user" +# Password to access the virtual administrative database admin_password = "admin_pass" -# pool -# configs are structured as pool. -# the pool_name is what clients use as database name when connecting -# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded_db" +# pool configs are structured as pool. +# the pool_name is what clients use as database name when connecting. +# For a pool named `sharded_db`, clients access that pool using connection string like +# `postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded_db` [pools.sharded_db] # Pool mode (see PgBouncer docs for more). -# session: one server connection per connected client -# transaction: one server connection per client transaction +# `session` one server connection per connected client +# `transaction` one server connection per client transaction pool_mode = "transaction" -# If the client doesn't specify, route traffic to -# this role by default. -# -# any: round-robin between primary and replicas, -# replica: round-robin between replicas only without touching the primary, -# primary: all queries go to the primary unless otherwise specified. +# Load balancing mode +# `random` selects the server at random +# `loc` selects the server with the least outstanding busy conncetions +load_balancing_mode = "random" + +# If the client doesn't specify, PgCat routes traffic to this role by default. +# `any` round-robin between primary and replicas, +# `replica` round-robin between replicas only without touching the primary, +# `primary` all queries go to the primary unless otherwise specified. default_role = "any" -# Query parser. If enabled, we'll attempt to parse +# If Query Parser is enabled, we'll attempt to parse # every incoming query to determine if it's a read or a write. # If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, # we'll direct it to the primary. @@ -93,23 +105,26 @@ primary_reads_enabled = true # So what if you wanted to implement a different hashing function, # or you've already built one and you want this pooler to use it? -# # Current options: -# -# pg_bigint_hash: PARTITION BY HASH (Postgres hashing function) -# sha1: A hashing function based on SHA1 -# +# `pg_bigint_hash`: PARTITION BY HASH (Postgres hashing function) +# `sha1`: A hashing function based on SHA1 sharding_function = "pg_bigint_hash" # Automatically parse this from queries and route queries to the right shard! -automatic_sharding_key = "data.id" +# automatic_sharding_key = "data.id" # Idle timeout can be overwritten in the pool idle_timeout = 40000 -# Credentials for users that may connect to this cluster +# Connect timeout can be overwritten in the pool +connect_timeout = 3000 + +# User configs are structured as pool..users. +# This secion holds the credentials for users that may connect to this cluster [pools.sharded_db.users.0] +# Postgresql username username = "sharding_user" +# Postgresql password password = "sharding_user" # Maximum number of server connections that can be established for this user # The maximum number of connection from a single Pgcat process to any database in the cluster @@ -117,6 +132,7 @@ password = "sharding_user" pool_size = 9 # Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. +# 0 means it is disabled. statement_timeout = 0 [pools.sharded_db.users.1] @@ -125,28 +141,26 @@ password = "other_user" pool_size = 21 statement_timeout = 15000 -# Shard 0 +# Shard configs are structured as pool..shards. +# Each shard config contains a list of servers that make up the shard +# and the database name to use. [pools.sharded_db.shards.0] -# [ host, port, role ] -servers = [ - [ "127.0.0.1", 5432, "primary" ], - [ "localhost", 5432, "replica" ] -] +# Array of servers in the shard, each server entry is an array of `[host, port, role]` +servers = [["127.0.0.1", 5432, "primary"], ["localhost", 5432, "replica"]] + +# Array of mirrors for the shard, each mirror entry is an array of `[host, port, index of server in servers array]` +# Traffic hitting the server identified by the index will be sent to the mirror. +# mirrors = [["1.2.3.4", 5432, 0], ["1.2.3.4", 5432, 1]] + # Database name (e.g. "postgres") database = "shard0" [pools.sharded_db.shards.1] -servers = [ - [ "127.0.0.1", 5432, "primary" ], - [ "localhost", 5432, "replica" ], -] +servers = [["127.0.0.1", 5432, "primary"], ["localhost", 5432, "replica"]] database = "shard1" [pools.sharded_db.shards.2] -servers = [ - [ "127.0.0.1", 5432, "primary" ], - [ "localhost", 5432, "replica" ], -] +servers = [["127.0.0.1", 5432, "primary" ], ["localhost", 5432, "replica" ]] database = "shard2" diff --git a/utilities/generate_config_docs.py b/utilities/generate_config_docs.py new file mode 100644 index 000000000..d02c88988 --- /dev/null +++ b/utilities/generate_config_docs.py @@ -0,0 +1,92 @@ +import re +import tomli + +class DocGenerator: + def __init__(self, filename): + self.doc = [] + self.current_section = "" + self.current_comment = [] + self.current_field_name = "" + self.current_field_value = [] + self.current_field_unset = False + self.filename = filename + + def write(self): + with open("../CONFIG.md", "w") as text_file: + text_file.write("# PgCat Configurations \n") + for entry in self.doc: + if entry["name"] == "__section__": + text_file.write("## `" + entry["section"] + "` Section" + "\n") + text_file.write("\n") + continue + text_file.write("### " + entry["name"]+ "\n") + text_file.write("```"+ "\n") + text_file.write("path: " + entry["fqdn"]+ "\n") + text_file.write("default: " + entry["defaults"].strip()+ "\n") + if entry["example"] is not None: + text_file.write("example: " + entry["example"].strip()+ "\n") + text_file.write("```"+ "\n") + text_file.write("\n") + text_file.write(entry["comment"]+ "\n") + text_file.write("\n") + + def save_entry(self): + if len(self.current_field_name) == 0: + return + if len(self.current_comment) == 0: + return + self.current_section = self.current_section.replace("sharded_db", "") + self.current_section = self.current_section.replace("simple_db", "") + self.current_section = self.current_section.replace("users.0", "users.") + self.current_section = self.current_section.replace("users.1", "users.") + self.current_section = self.current_section.replace("shards.0", "shards.") + self.current_section = self.current_section.replace("shards.1", "shards.") + self.doc.append( + { + "name": self.current_field_name, + "fqdn": self.current_section + "." + self.current_field_name, + "section": self.current_section, + "comment": "\n".join(self.current_comment), + "defaults": self.current_field_value if not self.current_field_unset else "", + "example": self.current_field_value if self.current_field_unset else None + } + ) + self.current_comment = [] + self.current_field_name = "" + self.current_field_value = [] + def parse(self): + with open("../pgcat.toml", "r") as f: + for line in f.readlines(): + line = line.strip() + if len(line) == 0: + self.save_entry() + + if line.startswith("["): + self.current_section = line[1:-1] + self.current_field_name = "__section__" + self.current_field_unset = False + self.save_entry() + + elif line.startswith("#"): + results = re.search("^#\s*([A-Za-z0-9_]+)\s*=(.+)$", line) + if results is not None: + self.current_field_name = results.group(1) + self.current_field_value = results.group(2) + self.current_field_unset = True + self.save_entry() + else: + self.current_comment.append(line[1:].strip()) + else: + results = re.search("^\s*([A-Za-z0-9_]+)\s*=(.+)$", line) + if results is None: + continue + self.current_field_name = results.group(1) + self.current_field_value = results.group(2) + self.current_field_unset = False + self.save_entry() + self.save_entry() + return self + + +DocGenerator("../pgcat.toml").parse().write() + From ac32e59087ba43cdbaff93ddaf960a619f699032 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 10 Mar 2023 22:02:33 -0600 Subject: [PATCH 351/551] Update README.md (#352) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ddbb661d4..f1303f02a 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ PGPASSWORD=postgres psql -h 127.0.0.1 -p 6432 -U postgres -c 'SELECT 1' ### Config -(See Configurations page)[https://github.com/levkk/pgcat/blob/main/CONFIG.md] +See [Configurations page](https://github.com/levkk/pgcat/blob/main/CONFIG.md) ## Local development From c77b8815ba0bac9aba61aa5201c66dc5e2ff9251 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sat, 11 Mar 2023 07:55:07 -0600 Subject: [PATCH 352/551] Update CONFIG.md (#353) Mark experimental features as such --- CONFIG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index b6dcc4780..188a2b137 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -205,7 +205,7 @@ If the client doesn't specify, PgCat routes traffic to this role by default. `replica` round-robin between replicas only without touching the primary, `primary` all queries go to the primary unless otherwise specified. -### query_parser_enabled +### query_parser_enabled (experimental) ``` path: pools..query_parser_enabled default: true @@ -226,7 +226,7 @@ If the query parser is enabled and this setting is enabled, the primary will be load balancing of read queries. Otherwise, the primary will only be used for write queries. The primary can always be explicitly selected with our custom protocol. -### sharding_key_regex +### sharding_key_regex (experimental) ``` path: pools..sharding_key_regex default: @@ -248,7 +248,7 @@ Current options: `pg_bigint_hash`: PARTITION BY HASH (Postgres hashing function) `sha1`: A hashing function based on SHA1 -### automatic_sharding_key +### automatic_sharding_key (experimental) ``` path: pools..automatic_sharding_key default: @@ -320,7 +320,7 @@ default: [["127.0.0.1", 5432, "primary"], ["localhost", 5432, "replica"]] Array of servers in the shard, each server entry is an array of `[host, port, role]` -### mirrors +### mirrors (experimental) ``` path: pools..shards..mirrors default: From c613293445ce13406e5d209539130f5222803677 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 15 Mar 2023 17:58:45 -0500 Subject: [PATCH 353/551] Reduce memory and CPU footprint of mirroring (#369) The experimental mirroring feature used a lot of memory and CPU when put under production traffic. This change attempts to reduce memory and CPU usage. Memory footprint is reduced by making the channel smaller. CPU usage is reduced by avoiding allocations if the channel is full or is closed. We might lose more messages this way if the mirror falls behind but that is more acceptable than crashing the entire process when it goes out-of-memory (OOM) --- src/mirrors.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/mirrors.rs b/src/mirrors.rs index 6a59172b7..ab2b2dc40 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -119,7 +119,7 @@ impl MirroringManager { let mut exit_senders: Vec> = vec![]; addresses.iter().for_each(|mirror| { - let (bytes_tx, bytes_rx) = channel::(500); + let (bytes_tx, bytes_rx) = channel::(10); let (exit_tx, exit_rx) = channel::<()>(1); let mut addr = mirror.clone(); addr.role = Role::Mirror; @@ -142,15 +142,25 @@ impl MirroringManager { } pub fn send(self: &mut Self, bytes: &BytesMut) { - let cpy = bytes.clone().freeze(); - self.byte_senders - .iter_mut() - .for_each(|sender| match sender.try_send(cpy.clone()) { + // We want to avoid performing an allocation if we won't be able to send the message + // There is a possibility of a race here where we check the capacity and then the channel is + // closed or the capacity is reduced to 0, but mirroring is best effort anyway + if self + .byte_senders + .iter() + .all(|sender| sender.capacity() == 0 || sender.is_closed()) + { + return; + } + let immutable_bytes = bytes.clone().freeze(); + self.byte_senders.iter_mut().for_each(|sender| { + match sender.try_send(immutable_bytes.clone()) { Ok(_) => {} Err(err) => { warn!("Failed to send bytes to a mirror channel {}", err); } - }); + } + }); } pub fn disconnect(self: &mut Self) { From abd814969478b44058cf6d7c7a40751f06419779 Mon Sep 17 00:00:00 2001 From: Fraser Isbester Date: Thu, 16 Mar 2023 14:07:02 -0600 Subject: [PATCH 354/551] github/workflows: adds automated image building (#370) * github/workflow: add ghcr build-push workflow * github/workflow: add build caching and push * github/workflows: add registry prefix * github/workflows: add build-concurrency groups w/ termination --- .github/workflows/build-and-push.yaml | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/build-and-push.yaml diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml new file mode 100644 index 000000000..1baa93bea --- /dev/null +++ b/.github/workflows/build-and-push.yaml @@ -0,0 +1,54 @@ +name: Build and Push + +on: push + +env: + registry: ghcr.io + image-name: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Determine tags + id: metadata + uses: docker/metadata-action@v4 + with: + images: ${{ env.registry }}/${{ env.image-name }} + tags: | + type=sha,prefix=,format=long + type=schedule + type=ref,event=branch + type=ref,event=pr + type=raw,value=latest,enable={{ is_default_branch }} + + - name: Log in to the Container registry + uses: docker/login-action@v2.1.0 + with: + registry: ${{ env.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push ${{ env.image-name }} + uses: docker/build-push-action@v3 + with: + push: true + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true From 7bb778d87d28e2d0f4faffd94a1b32fe1b9b2007 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 17 Mar 2023 12:31:43 -0500 Subject: [PATCH 355/551] Check Slice bounds in read_message to avoid panics (#371) When recv is called in the mirroring client, we noticed an occasional panic when reading the message. thread 'tokio-runtime-worker' panicked at 'slice index starts at 5 but ends at 0', src/messages.rs:522:18 We are still debugging the reason why this happens but adding a check for slice bounds seems like a good idea. Instead of panicking, this will return an Err to the caller which will close the connection. --- src/messages.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/messages.rs b/src/messages.rs index 3fc84b5a9..c9ace4e0b 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -517,14 +517,18 @@ where bytes.resize(bytes.len() + len as usize - mem::size_of::(), b'0'); - match stream - .read_exact( - &mut bytes[mem::size_of::() + mem::size_of::() - ..mem::size_of::() + mem::size_of::() + len as usize - - mem::size_of::()], - ) - .await - { + let slice_start = mem::size_of::() + mem::size_of::(); + let slice_end = slice_start + len as usize - mem::size_of::(); + + // Avoids a panic + if slice_end < slice_start { + return Err(Error::SocketError(format!( + "Error reading message from socket - Code: {:?} - Length {:?}, Error: {:?}", + code, len, "Unexpected length value for message" + ))); + } + + match stream.read_exact(&mut bytes[slice_start..slice_end]).await { Ok(_) => (), Err(err) => { return Err(Error::SocketError(format!( From cba373aa04ac31886c75eb5bf9d55e3fcc66e6c9 Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Fri, 24 Mar 2023 11:20:30 -0400 Subject: [PATCH 356/551] Add idle client in transaction configuration (#380) * Add idle client in transaction configuration * fmt * Update docs * trigger build * Add tests * Make the config dynamic from reloads * fmt * comments * trigger build * fix config.md * remove error --- CONFIG.md | 8 ++++++ pgcat.toml | 3 +++ src/client.rs | 24 +++++++++++++++--- src/config.rs | 26 ++++++++++++++++++++ tests/ruby/misc_spec.rb | 54 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 4 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 188a2b137..bcd6f09ff 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -49,6 +49,14 @@ default: 30000 # milliseconds How long an idle connection with a server is left open (ms). +### idle_client_in_transaction_timeout +``` +path: general.idle_client_in_transaction_timeout +default: 0 # milliseconds +``` + +How long a client is allowed to be idle while in a transaction (ms). + ### healthcheck_timeout ``` path: general.healthcheck_timeout diff --git a/pgcat.toml b/pgcat.toml index 7903cdfb5..0d883a332 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -23,6 +23,9 @@ connect_timeout = 5000 # milliseconds # How long an idle connection with a server is left open (ms). idle_timeout = 30000 # milliseconds +# How long a client is allowed to be idle while in a transaction (ms). +idle_client_in_transaction_timeout = 0 # milliseconds + # How much time to give the health check query to return with a result (ms). healthcheck_timeout = 1000 # milliseconds diff --git a/src/client.rs b/src/client.rs index bb70a6567..68f6f5751 100644 --- a/src/client.rs +++ b/src/client.rs @@ -12,7 +12,7 @@ use tokio::sync::broadcast::Receiver; use tokio::sync::mpsc::Sender; use crate::admin::{generate_server_info_for_admin, handle_admin}; -use crate::config::{get_config, Address, PoolMode}; +use crate::config::{get_config, get_idle_client_in_transaction_timeout, Address, PoolMode}; use crate::constants::*; use crate::messages::*; @@ -859,6 +859,11 @@ where let mut initial_message = Some(message); + let idle_client_timeout_duration = match get_idle_client_in_transaction_timeout() { + 0 => tokio::time::Duration::MAX, + timeout => tokio::time::Duration::from_millis(timeout), + }; + // Transaction loop. Multiple queries can be issued by the client here. // The connection belongs to the client until the transaction is over, // or until the client disconnects if we are in session mode. @@ -870,15 +875,26 @@ where None => { trace!("Waiting for message inside transaction or in session mode"); - match read_message(&mut self.read).await { - Ok(message) => message, - Err(err) => { + match tokio::time::timeout( + idle_client_timeout_duration, + read_message(&mut self.read), + ) + .await + { + Ok(Ok(message)) => message, + Ok(Err(err)) => { // Client disconnected inside a transaction. // Clean up the server and re-use it. server.checkin_cleanup().await?; return Err(err); } + Err(_) => { + // Client idle in transaction timeout + error_response(&mut self.write, "idle transaction timeout").await?; + error!("Client idle in transaction timeout: {{ pool_name: {:?}, username: {:?}, shard: {:?}, role: \"{:?}\"}}", self.pool_name.clone(), self.username.clone(), query_router.shard(), query_router.role()); + break; + } } } Some(message) => { diff --git a/src/config.rs b/src/config.rs index 5adf8d5e8..a3182e2be 100644 --- a/src/config.rs +++ b/src/config.rs @@ -197,6 +197,9 @@ pub struct General { #[serde(default = "General::default_ban_time")] pub ban_time: i64, + #[serde(default = "General::default_idle_client_in_transaction_timeout")] + pub idle_client_in_transaction_timeout: u64, + #[serde(default = "General::default_worker_threads")] pub worker_threads: usize, @@ -260,6 +263,10 @@ impl General { pub fn default_worker_threads() -> usize { 4 } + + pub fn default_idle_client_in_transaction_timeout() -> u64 { + 0 + } } impl Default for General { @@ -276,6 +283,7 @@ impl Default for General { healthcheck_delay: Self::default_healthcheck_delay(), ban_time: Self::default_ban_time(), worker_threads: Self::default_worker_threads(), + idle_client_in_transaction_timeout: Self::default_idle_client_in_transaction_timeout(), tcp_keepalives_idle: Self::default_tcp_keepalives_idle(), tcp_keepalives_count: Self::default_tcp_keepalives_count(), tcp_keepalives_interval: Self::default_tcp_keepalives_interval(), @@ -655,6 +663,13 @@ impl From<&Config> for std::collections::HashMap { config.general.healthcheck_delay.to_string(), ), ("ban_time".to_string(), config.general.ban_time.to_string()), + ( + "idle_client_in_transaction_timeout".to_string(), + config + .general + .idle_client_in_transaction_timeout + .to_string(), + ), ]; r.append(&mut static_settings); @@ -666,6 +681,10 @@ impl Config { /// Print current configuration. pub fn show(&self) { info!("Ban time: {}s", self.general.ban_time); + info!( + "Idle client in transaction timeout: {}ms", + self.general.idle_client_in_transaction_timeout + ); info!("Worker threads: {}", self.general.worker_threads); info!( "Healthcheck timeout: {}ms", @@ -819,6 +838,12 @@ pub fn get_config() -> Config { (*(*CONFIG.load())).clone() } +pub fn get_idle_client_in_transaction_timeout() -> u64 { + (*(*CONFIG.load())) + .general + .idle_client_in_transaction_timeout +} + /// Parse the configuration file located at the path. pub async fn parse(path: &str) -> Result<(), Error> { let mut contents = String::new(); @@ -889,6 +914,7 @@ mod test { assert_eq!(get_config().path, "pgcat.toml".to_string()); assert_eq!(get_config().general.ban_time, 60); + assert_eq!(get_config().general.idle_client_in_transaction_timeout, 0); assert_eq!(get_config().general.idle_timeout, 30000); assert_eq!(get_config().pools.len(), 2); assert_eq!(get_config().pools["sharded_db"].shards.len(), 3); diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index 2f69fb47d..1a04c5d67 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -309,4 +309,58 @@ end end end + + describe "Idle client timeout" do + context "idle transaction timeout set to 0" do + before do + current_configs = processes.pgcat.current_config + correct_idle_client_transaction_timeout = current_configs["general"]["idle_client_in_transaction_timeout"] + puts(current_configs["general"]["idle_client_in_transaction_timeout"]) + + current_configs["general"]["idle_client_in_transaction_timeout"] = 0 + + processes.pgcat.update_config(current_configs) # with timeout 0 + processes.pgcat.reload_config + end + + it "Allow client to be idle in transaction" do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("BEGIN") + conn.async_exec("SELECT 1") + sleep(2) + conn.async_exec("COMMIT") + conn.close + end + end + + context "idle transaction timeout set to 500ms" do + before do + current_configs = processes.pgcat.current_config + correct_idle_client_transaction_timeout = current_configs["general"]["idle_client_in_transaction_timeout"] + current_configs["general"]["idle_client_in_transaction_timeout"] = 500 + + processes.pgcat.update_config(current_configs) # with timeout 500 + processes.pgcat.reload_config + end + + it "Allow client to be idle in transaction below timeout" do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("BEGIN") + conn.async_exec("SELECT 1") + sleep(0.4) # below 500ms + conn.async_exec("COMMIT") + conn.close + end + + it "Error when client idle in transaction time exceeds timeout" do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("BEGIN") + conn.async_exec("SELECT 1") + sleep(1) # above 500ms + expect{ conn.async_exec("COMMIT") }.to raise_error(PG::SystemError, /idle transaction timeout/) + conn.async_exec("SELECT 1") # should be able to send another query + conn.close + end + end + end end From 7def2f7ad6a9e5df92278538a4332842bcb20ebf Mon Sep 17 00:00:00 2001 From: Montana Low Date: Fri, 24 Mar 2023 12:59:03 -0700 Subject: [PATCH 357/551] point CI at updated repo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1303f02a..dc42d0d5f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ##### PgCat: PostgreSQL at petabyte scale -[![CircleCI](https://circleci.com/gh/levkk/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/levkk/pgcat/tree/main) +[![CircleCI](https://circleci.com/gh/postgresml/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/postgresml/pgcat/tree/main) Join our Discord! From d6f851212f6e09b82e5c9723cf0f6fdc16c09944 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 24 Mar 2023 17:10:12 -0700 Subject: [PATCH 358/551] V1 (#383) --- .github/workflows/publish-ci-docker-image.yml | 6 +- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 388 ++++-------------- images/instacart.webp | Bin 0 -> 3448 bytes images/postgresml.webp | Bin 0 -> 4862 bytes 6 files changed, 79 insertions(+), 319 deletions(-) create mode 100644 images/instacart.webp create mode 100644 images/postgresml.webp diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml index 3cae8e7e2..5df4fd48d 100644 --- a/.github/workflows/publish-ci-docker-image.yml +++ b/.github/workflows/publish-ci-docker-image.yml @@ -15,6 +15,6 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build CI Docker image run: | - docker build . -f Dockerfile.ci --tag ghcr.io/levkk/pgcat-ci:latest - docker run ghcr.io/levkk/pgcat-ci:latest - docker push ghcr.io/levkk/pgcat-ci:latest + docker build . -f Dockerfile.ci --tag ghcr.io/postgresml/pgcat-ci:latest + docker run ghcr.io/postgresml/pgcat-ci:latest + docker push ghcr.io/postgresml/pgcat-ci:latest diff --git a/Cargo.lock b/Cargo.lock index 9856f6e09..9124996fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -716,7 +716,7 @@ dependencies = [ [[package]] name = "pgcat" -version = "0.6.0-alpha1" +version = "1.0.0" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 4be3c7670..e5b1de668 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "0.6.0-alpha1" +version = "1.0.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README.md b/README.md index dc42d0d5f..28433defb 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,47 @@ -##### PgCat: PostgreSQL at petabyte scale +## PgCat: Nextgen PostgreSQL Pooler [![CircleCI](https://circleci.com/gh/postgresml/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/postgresml/pgcat/tree/main) Join our Discord! -PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover support. - -**Beta**: looking for beta testers, see [#35](https://github.com/levkk/pgcat/issues/35). +PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load balancing, failover and mirroring. ## Features -| **Feature** | **Status** | **Comments** | -|--------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| -| Transaction pooling | :white_check_mark: | Identical to PgBouncer. | -| Session pooling | :white_check_mark: | Identical to PgBouncer. | -| `COPY` support | :white_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | -| Query cancellation | :white_check_mark: | Supported both in transaction and session pooling modes. | -| Load balancing of read queries | :white_check_mark: | Using random between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | -| Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | -| Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | -| Statistics | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. | -| Live configuration reloading | :white_check_mark: | Reload supported settings with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)` or `RELOAD` query issued to the admin database. | -| Client authentication | :white_check_mark: :wrench: | MD5 password authentication is supported, SCRAM is on the roadmap; one user is used to connect to Postgres with both SCRAM and MD5 supported. | -| Admin database | :white_check_mark: | The admin database, similar to PgBouncer's, allows to query for statistics and reload the configuration. | + +| **Feature** | **Status** | **Comments** | +|-------------|------------|--------------| +| Transaction pooling | **Stable** | Identical to PgBouncer with notable improvements for handling bad clients and abandoned transactions. | +| Session pooling | **Stable** | Identical to PgBouncer. | +| Multi-threaded runtime | **Stable** | Using Tokio asynchronous runtime, the pooler takes advantage of multicore machines. | +| Load balancing of read queries | **Stable** | Queries are automatically load balanced between replicas and the primary. | +| Failover | **Stable** | Queries are automatically rerouted around broken replicas, validated by regular health checks. | +| Admin database statistics | **Stable** | Pooler statistics and administration via the `pgbouncer` and `pgcat` databases. | +| Prometheus statistics | **Stable** | Statistics are reported via a HTTP endpoint for Prometheus. | +| Client TLS | **Stable** | Clients can connect to the pooler using TLS/SSL. | +| Client/Server authentication | **Stable** | Clients can connect using MD5 authentication, supported by `libpq` and all Postgres client drivers. PgCat can connect to Postgres using MD5 and SCRAM-SHA-256. | +| Live configuration reloading | **Stable** | Identical to PgBouncer; all settings can be reloaded dynamically (except `host` and `port`). | +| Sharding using extended SQL syntax | **Experimental** | Clients can dynamically configure the pooler to route queries to specific shards. | +| Sharding using comments parsing/Regex | **Experimental** | Clients can include shard information (sharding key, shard ID) in the query comments. | +| Automatic sharding | **Experimental** | PgCat can parse queries detect sharding keys automatically, and route queries to the route shard. | +| Mirroring | **Experimental** | Mirror queries between multiple databases in order to test servers with realistic production traffic. | + + +## Status + +PgCat is stable and used in production to serve hundreds of thousands of queries per second. Some features remain experimental and are being actively developed. They are optional and can be enabled through configuration. + +| | | +|-|-| +||| +| [Instacart](https://tech.instacart.com/adopting-pgcat-a-nextgen-postgres-proxy-3cf284e68c2f) | [PostgresML](https://postgresml.org/blog/scaling-postgresml-to-one-million-requests-per-second) | ## Deployment See `Dockerfile` for example deployment using Docker. The pooler is configured to spawn 4 workers so 4 CPUs are recommended for optimal performance. That setting can be adjusted to spawn as many (or as little) workers as needed. +A Docker image is available from `docker pull ghcr.io/postgresml/pgcat:latest`. See our [Github packages repository](https://github.com/postgresml/pgcat/pkgs/container/pgcat). + For quick local example, use the Docker Compose environment provided: ```bash @@ -39,9 +53,13 @@ PGPASSWORD=postgres psql -h 127.0.0.1 -p 6432 -U postgres -c 'SELECT 1' ### Config -See [Configurations page](https://github.com/levkk/pgcat/blob/main/CONFIG.md) +See **[Configuration](https://github.com/levkk/pgcat/blob/main/CONFIG.md)**. -## Local development +## Contributing + +The project is being actively developed and looking for additional contributors and production deployments. + +### Local development 1. Install Rust (latest stable will work great). 2. `cargo build --release` (to get better benchmarks). @@ -51,7 +69,7 @@ See [Configurations page](https://github.com/levkk/pgcat/blob/main/CONFIG.md) ### Tests -Quickest way to test your changes is to use pgbench: +When making substantial modifications to the protocol implementation, make sure to test them with pgbench: ``` pgbench -i -h 127.0.0.1 -p 6432 && \ @@ -61,36 +79,26 @@ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended See [sharding README](./tests/sharding/README.md) for sharding logic testing. -Run `cargo test` to run Rust tests. +Additionally, all features are tested with Ruby, Python, and Rust tests unit and integration tests. + +Run `cargo test` to run Rust unit tests. + +Run the following commands to run Ruby and Python integration tests: -Run the following commands to run Integration tests locally. ``` cd tests/docker/ docker compose up --exit-code-from main # This will also produce coverage report under ./cov/ ``` -| **Feature** | **Tested in CI** | **Tested manually** | **Comments** | -|-----------------------|--------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------| -| Transaction pooling | :white_check_mark: | :white_check_mark: | Used by default for all tests. | -| Session pooling | :white_check_mark: | :white_check_mark: | Tested by running pgbench with `--protocol prepared` which only works in session mode. | -| `COPY` | :white_check_mark: | :white_check_mark: | `pgbench -i` uses `COPY`. `COPY FROM` is tested as well. | -| Query cancellation | :white_check_mark: | :white_check_mark: | `psql -c 'SELECT pg_sleep(1000);'` and press `Ctrl-C`. | -| Load balancing | :white_check_mark: | :white_check_mark: | We could test this by emitting statistics for each replica and compare them. | -| Failover | :white_check_mark: | :white_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing is using Toxiproxy. | -| Sharding | :white_check_mark: | :white_check_mark: | See `tests/sharding` and `tests/ruby` for an Rails/ActiveRecord example. | -| Statistics | :white_check_mark: | :white_check_mark: | Query the admin database with `psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS'`. | -| Live config reloading | :white_check_mark: | :white_check_mark: | Run `kill -s SIGHUP $(pgrep pgcat)` and watch the config reload. | - -### Dev +### Docker-based local development -Also, you can open a 'dev' environment where you can debug tests easier by running the following command: +You can open a Docker development environment where you can debug tests easier. Run the following command to spin it up: ``` ./dev/script/console ``` -This will open a terminal in an environment similar to that used in tests. In there you can compile, run tests, do some debugging with the test environment, etc. Objects -compiled inside the contaner (and bundled gems) will be placed in `dev/cache` so they don't interfere with what you have in your host. +This will open a terminal in an environment similar to that used in tests. In there, you can compile the pooler, run tests, do some debugging with the test environment, etc. Objects compiled inside the contaner (and bundled gems) will be placed in `dev/cache` so they don't interfere with what you have on your machine. ## Usage @@ -105,11 +113,9 @@ In transaction mode, a client talks to one server for the duration of a single t This mode is enabled by default. ### Load balancing of read queries -All queries are load balanced against the configured servers using the random algorithm. The most straight forward configuration example would be to put this pooler in front of several replicas and let it load balance all queries. - -If the configuration includes a primary and replicas, the queries can be separated with the built-in query parser. The query parser will interpret the query and route all `SELECT` queries to a replica, while all other queries including explicit transactions will be routed to the primary. +All queries are load balanced against the configured servers using either the random or least open connections algorithms. The most straightforward configuration example would be to put this pooler in front of several replicas and let it load balance all queries. -The query parser is disabled by default. +If the configuration includes a primary and replicas, the queries can be separated with the built-in query parser. The query parser, implemented with the `sqlparser` crate, will interpret the query and route all `SELECT` queries to a replica, while all other queries including explicit transactions will be routed to the primary. #### Query parser The query parser will do its best to determine where the query should go, but sometimes that's not possible. In that case, the client can select which server it wants using this custom SQL syntax: @@ -136,38 +142,14 @@ The setting will persist until it's changed again or the client disconnects. By default, all queries are routed to the first available server; `default_role` setting controls this behavior. ### Failover -All servers are checked with a `SELECT 1` query before being given to a client. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the ban list is cleared: this is a safety precaution against false positives. The primary can never be banned. +All servers are checked with a `;` (very fast) query before being given to a client. Additionally, the server health is monitored with every client query that it processes. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the ban list is cleared: this is a safety precaution against false positives. The primary can never be banned. The ban time can be changed with `ban_time`. The default is 60 seconds. -Failover behavior can get pretty interesting (read complex) when multiple configurations and factors are involved. The table below will try to explain what PgCat does in each scenario: - -| **Query** | **`SET SERVER ROLE TO`** | **`query_parser_enabled`** | **`primary_reads_enabled`** | **Target state** | **Outcome** | -|---------------------------|--------------------------|----------------------------|-----------------------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Read query, i.e. `SELECT` | unset (any) | false | false | up | Query is routed to the first instance in the random loop. | -| Read query | unset (any) | true | false | up | Query is routed to the first replica instance in the random loop. | -| Read query | unset (any) | true | true | up | Query is routed to the first instance in the random loop. | -| Read query | replica | false | false | up | Query is routed to the first replica instance in the random loop. | -| Read query | primary | false | false | up | Query is routed to the primary. | -| Read query | unset (any) | false | false | down | First instance is banned for reads. Next target in the random loop is attempted. | -| Read query | unset (any) | true | false | down | First replica instance is banned. Next replica instance is attempted in the random loop. | -| Read query | unset (any) | true | true | down | First instance (even if primary) is banned for reads. Next instance is attempted in the random loop. | -| Read query | replica | false | false | down | First replica instance is banned. Next replica instance is attempted in the random loop. | -| Read query | primary | false | false | down | The query is attempted against the primary and fails. The client receives an error. | -| | | | | | | -| Write query e.g. `INSERT` | unset (any) | false | false | up | The query is attempted against the first available instance in the random loop. If the instance is a replica, the query fails and the client receives an error. | -| Write query | unset (any) | true | false | up | The query is routed to the primary. | -| Write query | unset (any) | true | true | up | The query is routed to the primary. | -| Write query | primary | false | false | up | The query is routed to the primary. | -| Write query | replica | false | false | up | The query is routed to the replica and fails. The client receives an error. | -| Write query | unset (any) | true | false | down | The query is routed to the primary and fails. The client receives an error. | -| Write query | unset (any) | true | true | down | The query is routed to the primary and fails. The client receives an error. | -| Write query | primary | false | false | down | The query is routed to the primary and fails. The client receives an error. | -| | | | | | | - ### Sharding We use the `PARTITION BY HASH` hashing function, the same as used by Postgres for declarative partitioning. This allows to shard the database using Postgres partitions and place the partitions on different servers (shards). Both read and write queries can be routed to the shards using this pooler. +#### Extended syntax To route queries to a particular shard, we use this custom SQL syntax: ```sql @@ -182,7 +164,8 @@ The active shard will last until it's changed again or the client disconnects. B For hash function implementation, see `src/sharding.rs` and `tests/sharding/partition_hash_test_setup.sql`. -#### ActiveRecord/Rails + +##### ActiveRecord/Rails ```ruby class User < ActiveRecord::Base @@ -210,7 +193,7 @@ User.connection.execute "SET SERVER ROLE TO 'auto'" User.find_by_email("test@example.com") ``` -#### Raw SQL +##### Raw SQL ```sql -- Grab a bunch of users from shard 1 @@ -230,268 +213,45 @@ SET SERVER ROLE TO 'auto'; -- let the query router figure out where the query sh SELECT * FROM users WHERE email = 'test@example.com'; -- shard setting lasts until set again; we are reading from the primary ``` -### Statistics reporting - -The stats are very similar to what Pgbouncer reports and the names are kept to be comparable. They are accessible by querying the admin database `pgcat`, and `pgbouncer` for compatibility. +#### With comments +Issuing queries to the pooler can cause additional latency. To reduce its impact, it's possible to include sharding information inside SQL comments sent via the query. This is reasonably easy to implement with ORMs like [ActiveRecord](https://api.rubyonrails.org/classes/ActiveRecord/QueryMethods.html#method-i-annotate) and [SQLAlchemy](https://docs.sqlalchemy.org/en/20/core/events.html#sql-execution-and-connection-events). ``` -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' -``` - -### Live configuration reloading - -The config can be reloaded by sending a `kill -s SIGHUP` to the process or by querying `RELOAD` to the admin database. Not all settings are currently supported by live reload: - -| **Config** | **Requires restart** | -|-------------------------|----------------------| -| `host` | yes | -| `port` | yes | -| `pool_mode` | no | -| `connect_timeout` | yes | -| `healthcheck_timeout` | no | -| `shutdown_timeout` | no | -| `healthcheck_delay` | no | -| `ban_time` | no | -| `user` | yes | -| `shards` | yes | -| `default_role` | no | -| `primary_reads_enabled` | no | -| `query_parser_enabled` | no | - - -## Benchmarks +/* shard_id: 5 */ SELECT * FROM foo WHERE id = 1234; -You can setup PgBench locally through PgCat: - -``` -pgbench -h 127.0.0.1 -p 6432 -i -``` - -Coincidenly, this uses `COPY` so you can test if that works. Additionally, we'll be running the following PgBench configurations: - -1. 16 clients, 2 threads -2. 32 clients, 2 threads -3. 64 clients, 2 threads -4. 128 clients, 2 threads - -All queries will be `SELECT` only (`-S`) just so disks don't get in the way, since the dataset will be effectively all in RAM. - -My setup: - -- 8 cores, 16 hyperthreaded (AMD Ryzen 5800X) -- 32GB RAM (doesn't matter for this benchmark, except to prove that Postgres will fit the whole dataset into RAM) - -### PgBouncer - -#### Config - -```ini -[databases] -shard0 = host=localhost port=5432 user=sharding_user password=sharding_user - -[pgbouncer] -pool_mode = transaction -max_client_conn = 1000 +/* sharding_key: 1234 */ SELECT * FROM foo WHERE id = 1234; ``` -Everything else stays default. +#### Automatic query parsing +PgCat can use the `sqlparser` crate to parse SQL queries and extract the sharding key. This is configurable with the `automatic_sharding_key` setting. This feature is still experimental, but it's the ideal implementation for sharding, requiring no client modifications. -#### Runs +### Statistics reporting +The stats are very similar to what PgBouncer reports and the names are kept to be comparable. They are accessible by querying the admin database `pgcat`, and `pgbouncer` for compatibility. ``` -$ pgbench -t 1000 -c 16 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 16 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 16000/16000 -latency average = 0.155 ms -tps = 103417.377469 (including connections establishing) -tps = 103510.639935 (excluding connections establishing) - - -$ pgbench -t 1000 -c 32 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 32 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 32000/32000 -latency average = 0.290 ms -tps = 110325.939785 (including connections establishing) -tps = 110386.513435 (excluding connections establishing) - - -$ pgbench -t 1000 -c 64 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 64 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 64000/64000 -latency average = 0.692 ms -tps = 92470.427412 (including connections establishing) -tps = 92618.389350 (excluding connections establishing) - -$ pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 128 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 128000/128000 -latency average = 1.406 ms -tps = 91013.429985 (including connections establishing) -tps = 91067.583928 (excluding connections establishing) +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' ``` -### PgCat +Additionally, Prometheus statistics are available at `/metrics` via HTTP. -#### Config +### Live configuration reloading -The only thing that matters here is the number of workers in the Tokio pool. Make sure to set it to < than the number of your CPU cores. -Also account for hyper-threading, so if you have that, take the number you got above and divide it by two, that way only "real" cores serving -requests. +The config can be reloaded by sending a `kill -s SIGHUP` to the process or by querying `RELOAD` to the admin database. All settings except the `host` and `port` can be reloaded without restarting the pooler, including sharding and replicas configurations. -My setup is 16 threads, 8 cores (`htop` shows as 16 CPUs), so I set the `max_workers` in Tokio to 4. Too many, and it starts conflicting with PgBench -which is also running on the same system. +### Mirroring -#### Runs +Mirroring allows to route queries to multiple databases at the same time. This is useful for prewarning replicas before placing them into the active configuration, or for testing different versions of Postgres with live traffic. +## License -``` -$ pgbench -t 1000 -c 16 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 16 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 16000/16000 -latency average = 0.164 ms -tps = 97705.088232 (including connections establishing) -tps = 97872.216045 (excluding connections establishing) - - -$ pgbench -t 1000 -c 32 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 32 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 32000/32000 -latency average = 0.288 ms -tps = 111300.488119 (including connections establishing) -tps = 111413.107800 (excluding connections establishing) - - -$ pgbench -t 1000 -c 64 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 64 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 64000/64000 -latency average = 0.556 ms -tps = 115190.496139 (including connections establishing) -tps = 115247.521295 (excluding connections establishing) - -$ pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 128 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 128000/128000 -latency average = 1.135 ms -tps = 112770.562239 (including connections establishing) -tps = 112796.502381 (excluding connections establishing) -``` +PgCat is free and open source, released under the MIT license. -### Direct Postgres +## Contributors -Always good to have a base line. +Many thanks to our amazing contributors! -#### Runs + + + -``` -$ pgbench -t 1000 -c 16 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 16 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 16000/16000 -latency average = 0.115 ms -tps = 139443.955722 (including connections establishing) -tps = 142314.859075 (excluding connections establishing) - -$ pgbench -t 1000 -c 32 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 32 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 32000/32000 -latency average = 0.212 ms -tps = 150644.840891 (including connections establishing) -tps = 152218.499430 (excluding connections establishing) - -$ pgbench -t 1000 -c 64 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 64 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 64000/64000 -latency average = 0.420 ms -tps = 152517.663404 (including connections establishing) -tps = 153319.188482 (excluding connections establishing) - -$ pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 128 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 128000/128000 -latency average = 0.854 ms -tps = 149818.594087 (including connections establishing) -tps = 150200.603049 (excluding connections establishing) -``` diff --git a/images/instacart.webp b/images/instacart.webp new file mode 100644 index 0000000000000000000000000000000000000000..a3aa1fca560ae0f6b6af8ec1aea6b25d02ebc223 GIT binary patch literal 3448 zcmaJ@c{o*DA6^_o8A1qUI5JckMCN18Sm=;h9F8foPGpLM%8+Cx2^mY79YeZxj3q>v`5*?|y&pyY_GW_S*YdCi*%$zN`RPXltU)(F&GK z001Q1cL0b3K>zY(^mAJH7SKNTbo9DScM$+DZ$F%gt|rpb${NYI01$u|@W6so4vx2d zHPC2-1NHxITVDXpL1gg!!Ce1a^N%d6lk;sy0BDe~Lfz39=LchZ7|R9t`5xdWFy?Z+ z>EHz8QW&4Z!3n~6^nl0yiI)$s>o5FkQCv7vlgsehxL_XX`ZtXI8+N>j^MW-putwC$ z%Nw4b*78s6bb#+3U@uR9IJaNzV1+Es-e%X~Q55d{Ko{r(Lx6_MiUt0_4R``Sa2_7L zVdge4f#aJ0;6MJ$kA_`X*yRQsVTTs*0bao2mmeIw16TvMziNBi<&yL-iiSlK0QxBi z`Y8qgV+sJ2L2L&Sln8LL(=m8+}DFEWG@Z2~4 zVcmyd!+-1ce>MN9AG~vsui&j)sR*)ZB*@?qP?h41(>f`f$6Q zXStuvq+0JGZG}3*PRz2L5(uEy^!0A7x#BZa!qtq39eAUCr`4`K;qM5t#b&}TenzEG z&6YB>3-G4U`6d13)SrWZsF7+P@r_|dRI^(<-Hb1J3a8INYLuF7wk)AiwF&547#IF5 zBt(wx@laQr#|%O<(5?N#T@&hch=+Vl8IuM*gnFpD-$cmr8_-tnvuWNb@0`zSm9w<_ zrPUX>)3sZwI~vTbcogCtlX#fAgix-b7tt;JXBaCg#^?C%b(cz|T~DajBVN&$sQ-?0 z-O;J2z$db6*kOA=#LO+|iL)0;FXcqW+>DN-9M4b2?EEJ9QtS$Lq=J_K98JrA$`~By z=#!XZ(ku=#R0uJzSbl6WW``2mr?FG<*jZKaL8M1VXGewKKeG48F8XQyhn=>U=#}i; z0a4-pagh{_%EZFmkYszN_XF<-r8s>uSMA=2%7n_1YeiTUwdV;rnV~8?i!pHi%IfsL z@^|i_>#$-Lb~^VURUJ^FFY}*^*s5~`rt}V9a3z|_dfsc*#7Z?0)M7ou*V^jT1Aezb zlKN6^z#lwHf@@Q6eEUnByoCQN4BxD3$=AoLT{3suYmZdlvEhnYA90iISIg88AO_26@!URB(|kUK8!!s$b!{Ax_yFR6xFJ;Y$t^UK$M3P7#g zXT={bew`ni+{v+kb68fntkkBIQ$B`_#S$cLpkl-b*bGg=cY-hp#32>%mv}P zX6kAf+f3H<$q|wqS4L7r#8S+?Wy+l0rRLVU)$UabYS$wngx*K>!TN_suRIjLWwS>s z{{kb7FmEsJU@X`2xAru-ui1{i$CIP{f(qHw9}&2vY~7bTZPbmo{_yTFqw{HDnnYiO z23bwmxmoopZ+)!bWa(7y4tlo5yRwh`d7)7+GdO+4jhOPi#sX!H9O0N{Rw++Q5*B}> z9jN-8R&kDx_|V~Wc2UL@^{vrcvo>b}X$yhMxG*2WntJHjQRZDJ_s{QT4p-=`6y z?;GbyYyvgYZPA4|ul=1$)6VmqXu6MdCYVH~ zv#LGJ4?Ro#Jc7eZouhd2P+Gn2CqrHZE;e+3w(NYdfxp`C`}$kVVu9A2QXkK?^Xu0f zpbSGf^OcgTDHyRTS6`p+M?INJ*Br?jb+Q)oO!4cPg8akuGYKn7T_b52#^i;_lkJi8 zYDH3R`^;6GuZ0?-7s}v2i2>)%Fb46Fyj>&{KyYu=l{J4_gRb5($5Gi2ec=Y5550U1-8?!`>GB%UHIn|K$dG;lt&!~2s-G%qx=_t= z^Y{A4Sc{&JdNb=6N6ZO{lBsuY_3aN>v^};y#LP{SIWd$h!JKm;xVf(Ljc}T|Ypl0z zYpiDEJKEui6xFHMy>y1NoHOq14Yd^25*^3pFmc{kt0D%`HZE>=O`N^?S#OoLkF4(( zb*nxHJmz=!ZU4u%OE7mj(UKZWxE^k7IwWZR$tRn4W*4ZBZQM67A3x?fd`QQGkf2lXH7x3kS3a#M?p;dsW0u~R z&5d)jPhP*o4Er;w5+;HSAtjqW6m;4_YWyzr?X{fX_MFk2-BNI{H@@|n_# zQJP+<+TMbstx&qUo5JbZQBfuAC5{d6NJJ0L?+%{mK|YjbK*-*sem&~+D@#_xYD z5Q5nRuxmtRXC?O3f*&2Wi{5vRXi1Ldt4?WM3GU$K%M9WQTcKxZ<77Y7yX-i;OnRd9 zNr1%HD)0Do>O`O0$#{bKHU3<&jok1XsDc%$YuHM?hsPa=YR1@NbWTcFR3?BHYA@(m zZF^asPIcp{QY{jEdYJ03`y|OB3`^^=d=ER{ajHU5H`E;Hv znR?po_qfk#{}Ol4x32!-j#%5nRi092Y(qLDRJSAAejwgz%(5*;*+mYKlQZI37dk0v zHqs;}?MX&^dD6bov-ScmCT{m&s+^96*4FA+wXJ*>^tfPrzLq@?Db`wSSt`-+D#X5f zQr6;Pgy6i*v*FJi`vUFDIZ*r3UQOBz!D8_&Ca;eAO548W)9N1{t3r+ZHUv z#<5(ToK#3V^8hsQ#f49IxGbU}^e^-Ccp6xiBoms6xV_Bt`!ul;1#-+iC zEqn&aNL{-41Dj1S%`u$BP2&aGi4(p0Au23hH&rGevZ)Yjjx8;tXc4*Pan2hbe`Iyj zfa)Z_duQ1XFWe)ijH^Q>Y5Ps*JGNfG&kk9mp8vw5%iLu?6LufVM2MD6-5 z61#jXaB;$hW_#F!{m#Y$qe1Lb!YJ{hcHAbH zfl;!~QD{2pc2{xFj!cyJX(C5VoN(>nNXcQ5Ci&jqmhW$MI@eY-G7K;$SXDg8e|t}! zIey%>Nr8Gd!27Aw5nmc^cD*tzj&bH6HG|>j`PflSB(ug)X`>6WkJsT0hw$(1gz?KFy4=84c(g)y6Uk1BQ~b`WgLi z>}Z5cBTQ^~P^v*MLg?l2qBEO%I?L6j7P!DS~bTI=5TbKh&P=UG4YzP7%`qesUy0AQ@Dq-&%rX+j17fDo=* z0YE$eXecY|7USb?0sLZD8;BPnE(C#kd+Mtxv74AaWhdDH2ml7)7EZupW#i?psH>}a zrT*{jcmlw?ihLKiTI;`S{*|S)wezw806cb_Lczw}(;J5^aai2X+x-e><1np_qm?ZV z*WfU}CvJl{Jaxre|BZL9u)|;YucT-_4fU0AxzXZyc8CAK*8hQR96cd8jVMk7vV}l# z>*JgJjcu=R&=rQb`r!8VSG&rP!VU^Hz|A0BX9m;&4L}Rf1=s;=zz1*wTmf%D05?N% zj2EDfi!1$$pYbohF3x3*b2$MvIEM=020(DfbL9uF&H$&uwSQ{sWiRsJFA9%B2>^%| zu-GFm03b;N08BC#dtQRYV&DKkFb4pg(0}}H#Q-4r8^^z;T4@deKu#_I za691Ee)cbUU&9&xuWtXZ=6~y_sHv;2j&tD^f5bF8HJeXJZwwd2?v_Ck-t2zXfDj^A z#v34NxbgAsRD)DuvtKYw(xDc4oWn$YQ=wTnyA9$fw|0C%wl5gfEHHPic(c4Mzkv`q zl7M)!gi>N2G7E358~t$6V!r35laojgC%>ImCAURGa?+k_PRgePI!ic&E z(gKhr@a4kO{D`of>i3v0_mbOvzy98MM()+In(If5W~0+PmL#F(X|;}YCrr9zgKRD) zwEVF`y%fcVA038gBz}E6Ii*Oa2n~}^B4DHx0uY_%9iC$Zw^;S%S+@0ASQjV^h)Z;ldVh{$&-*P-rpF zOR2yUK||!5Lj!^OdGOK$YBTr4{f0O>TCR-X-Kx?SB;%A9^RPhy9ztF_<{Sd%;M{mO z@LiOzihHNDDx&-|y*m=ICYfJa0tT{GDQpM%s);(8n{~vfwyPR-wwgvl&i^#;z1iZc zlC(N-tG?8?BPL$Gp6AUb72>P6+yTbC$$h+Ce4ybMf{a#m?Se~HO;?foyhpx}6)T!v z3T}38u4<>IUSrwc_tgF@t@Wb5ysd+U`n67>Nk*hyE_a|`^Q4UFpB z8+h*A%g;ty_etbh#Ml|@nZs!F8@j*9T6hKI=kYOtLMy+W)I|FDIqyvZ0yK(U;Me$E z1d>`JipJR#@SBbGL<9<-I^urvm?xEuDx&zUZ;`~gXY47NTlB08913jy+%*zRsWWPZ zmaXqo8STBR+_qAhf0B6S++xeJ^Fq_Zkrci$HHtJ%LlWg)j()80r+H6xI|`lsP;rKX z7#dCiRwuj zfPG@1nV~F_VCoRE(6Fy3-L!h8B}|BON791yZ$UJb&XBq$9YxA@_73NM9qb-nsMuRj zmyQRs@kG?PKs8~v8v&7zMW684R#>QZte2enS*c}nYr{*jtjfAz&=wKA8A(Z;S6mdA z+crq{kh4TDwa%u0Urs#0R86r+zOGJ2MIi1DMJSqk(2v7zG8dmk&wn`^8E&ej`bz^O z%?UZIQ2!#!A9ELmjYkKc4oN(3-xy~Ep*KwCviT!+@Az7Urv!HmWba1V&eSP^R@;Zt zRFhQVUh&M@Oc6JUSVe&qrcG^wNNXm4XwU~Ijn(&`;Q_AU9HcTQJqG<3oY`)U$^1oK zAAS9WjK}A8tlT!3zopXcSOd??J(c|&r>WOao;_m1{J;}Lg%Hm}Ive?Fi~HG2r05FI z&Iivg6q4H%4BTo3ZKFZ&ehU_r+eO`Wt7J|`{GupM+u>I??WRe8P7m4+0s01^)YA@7 zN*Zq~u22zMn=bAeh2yC3-d_U5p*HYviHLNSEJv!({35COu}7E%=u62MCNv(p|~GX8ecMtrg{L!ueS-?`RP* zJmPoY#5Q#giVwfPV(rF5-53Oh>0||xAB0Nn#@=nJn(7i(IF(T&7PrFYP`HC;%tVJk zzF@OEa5K4HCa{jkDomGFp3HVy)6|HRerg zG4#F@LZ+L$^PGcPFKL-U%y=*N98)$@aoJMGPtGCZJD$nhNo__<6*Kx%Gg7SMRvE}a zapXFH+ExpnyrX1<_-32B0A>Fuq2zOGF!eOXF~7nG$vG~V)^x4x*S^+?Rz6#tE{@De>?AjwO33)L)Y|)UTOP-SyQWLLV}7#sZBTZ#dCLylOR)( z=L5q&LOWOwS!Wv=d4bmNOSaT~!D_=x7AC%2{fiUX9cac?)sDK%Wi;W#WoZdvWsSmx zj8MgQUd%f#rj)9)k6}4Vld9Lz8!2^cgA6s60MpSkWtv0CV-fsy=GYMRJ(5(tiNp`J z;cp`eh|r)FZG9{tVYUkeoeYNKA$BJ5!Cj1DY2cN9bGPWI5C;c+iNVeB92>&uo=mQU zufSN*HMEFw*Y2YMo__b3IB*Ut=t2z7;7Gv2Xp4t99CMG5qT#fj!}QX&3ogr|?oSfr zY3t!tM
bo(a<=FaqB>4pK zKv3S7J?X7UcU)b(`ZEXmhxGILI&$aTK@M&-(KekPH7iam{pdts%62OZ#5ROj52$## z8o-=TA>*|T=lCe)``A;Sz)B}321N`pvuu)D` zjh$rSDN+8UbP`CE)t29+L`h^J8Nfe-aWoifFsFpr_nb#O)$??FrSA7|BLCMP#R0RB zKujFQAbs-y{`F#s@=M+KEZF7gIhDa;;NGhzI*#V?sn{aR)9-50>1pvczC&#rF`{eR zv7fLVvfE9>>4SKZ*^qgwbq&6yx<(wVX$emTD?vi)&jGLRYd1qEPhD(!ghq|l#hZjU~gl* z&XZ6i0c;Z38u?AwhfC~57Z#2Q{ljxMxr28xc?PRlJae;>pIx-1nu38B4`F8?KLwG8 z48Yd0vip2zC~WH#!6@!}>##Y3QXCLMuw(*7TcR$kB%Pmsa6uEr)!Rwfc|VKn>8B(3 zDw3nLornx4hc)Hk0nvh{nfLtD*f3bMnQQ9CqhL)Sg6|~{CVU}hUU8|?; z6V**I0@A#SV*=&!CzQK?z1oV#1f1si?*7XC42i5!v! z!P`#O*_-A2?eOkd<#EV|n|br^k$YwN0qFp6xz#P}n@@=?UY~85*BP}5Kb<}4W!y~M zp}QaNS(sTksou$U+UcLMq#zbv#dP-(#6-p-=sgW~-RkbIy5w8a8$1^*Fml5C{RW+Sj3qI=38f!IT z_I4x%Y#vCdG)Su88})S}b``ACLeC8=`wyXuucil#JSq2g$gQP)S9E0K$5~i>r3aIy zb|Q}-RR`+G$*c5cA8xf#%daC)6~7BvyI2V6%NP83C?paqDlL-3C6t+$u7eANzRhI{ z_h^G?_XrcWR9RBVPl_}T>SM{V(U0$RRoHq@ zdbzM>jEQEyF*%dH{9=(I;pv6GOg_|KpKGw7D2UFx-F%$6gQu9IDxK}d6AR0GE;Lfp zanj9pb#OGX^+@e6h>z&D#}tcO?8K+SLWS^vmQ;qgg-yQe;t|i6`sMB*twoocx?uNi z8vhAr?$US?h}sVsS4462t^mb1w1ezo@B!IKcfD!WIoFg<8m; z2(!wU!|vtnyxfQPp~r=W3<@ZZZ>2>kN7hrBF6Qraup$yQQ?3W=3WHQdYkW_gWxOTN z2|*rv4|zS1$XgeDRwrHxV^vfh1>K0N$VtfYu=_M;BJRg;p`p9x z;ya_)98^lMmjT0KJOhL9r}Z&k=S>hM1lFOqseJGc$_4feGxrioIZ4TbF4hpAKB`aXUXu%CYh1fJcBg4{=jelWi^x1R{}FK$!c)g zjAL)po-jpxc=r$G^^RU;vC9@&6Kz9>Gp0bUo=>&TmW-t?$7?IyWn_i=IX@1xDxd{T zkIHIakYf@!Dcui_N*lZ~NBOJBnLp2vGSI+S>!pwPM|v)fT3$@gW0n8u27Gs)7H6&2`dQs0&o1$^Ysyj?jUDv7j6b({*^E5;Z>s)!_+hwE%U z8A4o3>!?Qf0hq^V2l&a3cbZFL5AOXa&mh7y!t*(Vm{ z;Kc-utR*DSIFpPDTQ%R$AtS~SZc8T`6gqW9J6QyQEcdcrj+HaOd4RzsRunArAsVF+^VO=?u`4MS(%jP(F>VgrQa&R?2|sB@($LJZWvWz;MUu#!G3}0DB#t z%gcQ?Mlmeh^N2sB=Ysi9uig=sZ>eHEY?^QW33^}zZ2#i1`|@o1h@qbaD8HO*1|z4Z N+k2n#`0B63{{UIVm}>w4 literal 0 HcmV?d00001 From 0ba14835b46ef82589efdf442915400c26eea71a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 24 Mar 2023 17:12:12 -0700 Subject: [PATCH 359/551] Contributors (#384) --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 5cd126175..57e9e3831 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2022 Lev Kokotov +Copyright (c) 2023 PgCat Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the From b28ac5be3e9176df17ceaab1ccaa756d9b321899 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 26 Mar 2023 00:33:05 -0700 Subject: [PATCH 360/551] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 28433defb..4d6f599d2 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load bal | Live configuration reloading | **Stable** | Identical to PgBouncer; all settings can be reloaded dynamically (except `host` and `port`). | | Sharding using extended SQL syntax | **Experimental** | Clients can dynamically configure the pooler to route queries to specific shards. | | Sharding using comments parsing/Regex | **Experimental** | Clients can include shard information (sharding key, shard ID) in the query comments. | -| Automatic sharding | **Experimental** | PgCat can parse queries detect sharding keys automatically, and route queries to the route shard. | +| Automatic sharding | **Experimental** | PgCat can parse queries, detect sharding keys automatically, and route queries to the correct shard. | | Mirroring | **Experimental** | Mirror queries between multiple databases in order to test servers with realistic production traffic. | @@ -79,7 +79,7 @@ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended See [sharding README](./tests/sharding/README.md) for sharding logic testing. -Additionally, all features are tested with Ruby, Python, and Rust tests unit and integration tests. +Additionally, all features are tested with Ruby, Python, and Rust unit and integration tests. Run `cargo test` to run Rust unit tests. From f5d1940f15cb5af71a7dc276e932d56d9c47ea3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 09:12:43 -0700 Subject: [PATCH 361/551] chore(deps): bump regex from 1.7.1 to 1.7.3 (#385) Bumps [regex](https://github.com/rust-lang/regex) from 1.7.1 to 1.7.3. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.7.1...1.7.3) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9124996fe..367aa1ede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -871,9 +871,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" dependencies = [ "aho-corasick", "memchr", @@ -882,9 +882,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "ring" From a23a1c39f7d36f1c18a3cf17b793302452cc9d92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 09:12:54 -0700 Subject: [PATCH 362/551] chore(deps): bump async-trait from 0.1.66 to 0.1.68 (#382) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.66 to 0.1.68. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/compare/0.1.66...0.1.68) --- updated-dependencies: - dependency-name: async-trait dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 367aa1ede..0685e1565 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,13 +28,13 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-trait" -version = "0.1.66" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.9", ] [[package]] @@ -175,7 +175,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 1.0.109", ] [[package]] @@ -192,7 +192,7 @@ checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -308,7 +308,7 @@ checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -782,7 +782,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -814,18 +814,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -972,7 +972,7 @@ checksum = "4fc80d722935453bcafdc2c9a73cd6fac4dc1938f0346035d84bf99fa9e33217" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1094,6 +1094,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da4a3c17e109f700685ec577c0f85efd9b19bcf15c913985f14dc1ac01775aa" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "termcolor" version = "1.2.0" @@ -1157,7 +1168,7 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1339,7 +1350,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -1361,7 +1372,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] From 0dd1394f9af5e09da93748d9adc51fcc3ac151bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 09:13:13 -0700 Subject: [PATCH 363/551] chore(deps): bump serde from 1.0.154 to 1.0.158 (#376) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.154 to 1.0.158. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.154...v1.0.158) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0685e1565..df22b520c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -960,9 +960,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.154" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cdd151213925e7f1ab45a9bbfb129316bd00799784b174b7cc7bcd16961c49e" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" [[package]] name = "serde_derive" From 14a78e3bf959da5547d9fe69c4ef283e591fb69a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 09:13:25 -0700 Subject: [PATCH 364/551] chore(deps): bump toml from 0.7.2 to 0.7.3 (#360) Bumps [toml](https://github.com/toml-rs/toml) from 0.7.2 to 0.7.3. - [Release notes](https://github.com/toml-rs/toml/releases) - [Commits](https://github.com/toml-rs/toml/compare/toml-v0.7.2...toml-v0.7.3) --- updated-dependencies: - dependency-name: toml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df22b520c..59fd21ca1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1198,9 +1198,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6" +checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" dependencies = [ "serde", "serde_spanned", @@ -1219,9 +1219,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.4" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1eb0622d28f4b9c90adc4ea4b2b46b47663fde9ac5fafcb14a1369d5508825" +checksum = "08de71aa0d6e348f070457f85af8bd566e2bc452156a423ddf22861b3a953fae" dependencies = [ "indexmap", "serde", From 757c8fe24ace678c589d67169438377e683787b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 09:13:36 -0700 Subject: [PATCH 365/551] chore(deps): bump hyper from 0.14.24 to 0.14.25 (#358) Bumps [hyper](https://github.com/hyperium/hyper) from 0.14.24 to 0.14.25. - [Release notes](https://github.com/hyperium/hyper/releases) - [Changelog](https://github.com/hyperium/hyper/blob/v0.14.25/CHANGELOG.md) - [Commits](https://github.com/hyperium/hyper/compare/v0.14.24...v0.14.25) --- updated-dependencies: - dependency-name: hyper dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59fd21ca1..5af6090e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -453,9 +453,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.24" +version = "0.14.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" dependencies = [ "bytes", "futures-channel", From e269fcbfcab86c518ed3ebb9cb2777a8bec174e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 09:13:45 -0700 Subject: [PATCH 366/551] chore(deps): bump futures from 0.3.26 to 0.3.27 (#356) Bumps [futures](https://github.com/rust-lang/futures-rs) from 0.3.26 to 0.3.27. - [Release notes](https://github.com/rust-lang/futures-rs/releases) - [Changelog](https://github.com/rust-lang/futures-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/futures-rs/compare/0.3.26...0.3.27) --- updated-dependencies: - dependency-name: futures dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5af6090e4..151b00883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -254,9 +254,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" +checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" dependencies = [ "futures-channel", "futures-core", @@ -269,9 +269,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" dependencies = [ "futures-core", "futures-sink", @@ -279,15 +279,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" [[package]] name = "futures-executor" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" +checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" dependencies = [ "futures-core", "futures-task", @@ -296,15 +296,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" [[package]] name = "futures-macro" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" dependencies = [ "proc-macro2", "quote", @@ -313,21 +313,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" [[package]] name = "futures-task" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" [[package]] name = "futures-util" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" dependencies = [ "futures-channel", "futures-core", From f4011da52e9dc986f72ad08dc162ab812be74215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Tue, 28 Mar 2023 17:19:37 +0200 Subject: [PATCH 367/551] Refactor stats to use atomics (#375) * Refactor stats to use atomics When we are dealing with a high number of connections, generated stats cannot be consumed fast enough by the stats collector loop. This makes the stats subsystem inconsistent and a log of warning messages are thrown due to unregistered server/clients. This change refactors the stats subsystem so it uses atomics: - Now counters are handled using U64 atomics - Event system is dropped and averages are calculated using a loop every 15 seconds. - Now, instead of snapshots being generated ever second we keep track of servers/clients that have registered. Each pool/server/client has its own instance of the counter and makes changes directly, instead of adding an event that gets processed later. * Manually mplement Hash/Eq in `config::Address` ignoring stats * Add tests for client connection counters * Allow connecting to dockerized dev pgcat from the host * stats: Decrease cl_idle when idle socket disconnects --- Cargo.lock | 12 + Cargo.toml | 1 + dev/docker-compose.yaml | 2 + src/admin.rs | 121 +++-- src/client.rs | 155 ++++-- src/config.rs | 42 +- src/main.rs | 5 +- src/mirrors.rs | 26 +- src/pool.rs | 122 +++-- src/prometheus.rs | 59 ++- src/server.rs | 30 +- src/stats.rs | 1021 +++---------------------------------- src/stats/address.rs | 149 ++++++ src/stats/client.rs | 182 +++++++ src/stats/pool.rs | 274 ++++++++++ src/stats/server.rs | 226 ++++++++ tests/ruby/admin_spec.rb | 43 +- tests/ruby/capture | Bin 0 -> 40638 bytes tests/ruby/spec_helper.rb | 7 + 19 files changed, 1299 insertions(+), 1178 deletions(-) create mode 100644 src/stats/address.rs create mode 100644 src/stats/client.rs create mode 100644 src/stats/pool.rs create mode 100644 src/stats/server.rs create mode 100644 tests/ruby/capture diff --git a/Cargo.lock b/Cargo.lock index 151b00883..7e4aa6838 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,17 @@ dependencies = [ "syn 2.0.9", ] +[[package]] +name = "atomic_enum" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6227a8d6fdb862bcb100c4314d0d9579e5cd73fa6df31a2e6f6e1acd3c5f1207" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -720,6 +731,7 @@ version = "1.0.0" dependencies = [ "arc-swap", "async-trait", + "atomic_enum", "base64", "bb8", "bytes", diff --git a/Cargo.toml b/Cargo.toml index e5b1de668..3f0cbec64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ exitcode = "1.1.2" futures = "0.3" socket2 = { version = "0.4.7", features = ["all"] } nix = "0.26.2" +atomic_enum = "0.2.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index da759383f..15621e87b 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -26,6 +26,8 @@ x-common-env-pg: services: main: image: kubernetes/pause + ports: + - 6432 pg1: <<: *common-definition-pg diff --git a/src/admin.rs b/src/admin.rs index feea3a153..03af755c8 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,10 +1,11 @@ use crate::pool::BanReason; -/// Admin database. use bytes::{Buf, BufMut, BytesMut}; use log::{error, info, trace}; use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; use std::collections::HashMap; +/// Admin database. +use std::sync::atomic::Ordering; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::time::Instant; @@ -12,9 +13,7 @@ use crate::config::{get_config, reload_config, VERSION}; use crate::errors::Error; use crate::messages::*; use crate::pool::{get_all_pools, get_pool}; -use crate::stats::{ - get_address_stats, get_client_stats, get_pool_stats, get_server_stats, ClientState, ServerState, -}; +use crate::stats::{get_client_stats, get_pool_stats, get_server_stats, ClientState, ServerState}; use crate::ClientServerMap; pub fn generate_server_info_for_admin() -> BytesMut { @@ -158,7 +157,14 @@ where "free_clients".to_string(), client_stats .keys() - .filter(|client_id| client_stats.get(client_id).unwrap().state == ClientState::Idle) + .filter(|client_id| { + client_stats + .get(client_id) + .unwrap() + .state + .load(Ordering::Relaxed) + == ClientState::Idle + }) .count() .to_string(), ])); @@ -166,7 +172,14 @@ where "used_clients".to_string(), client_stats .keys() - .filter(|client_id| client_stats.get(client_id).unwrap().state == ClientState::Active) + .filter(|client_id| { + client_stats + .get(client_id) + .unwrap() + .state + .load(Ordering::Relaxed) + == ClientState::Active + }) .count() .to_string(), ])); @@ -178,7 +191,14 @@ where "free_servers".to_string(), server_stats .keys() - .filter(|server_id| server_stats.get(server_id).unwrap().state == ServerState::Idle) + .filter(|server_id| { + server_stats + .get(server_id) + .unwrap() + .state + .load(Ordering::Relaxed) + == ServerState::Idle + }) .count() .to_string(), ])); @@ -186,7 +206,14 @@ where "used_servers".to_string(), server_stats .keys() - .filter(|server_id| server_stats.get(server_id).unwrap().state == ServerState::Active) + .filter(|server_id| { + server_stats + .get(server_id) + .unwrap() + .state + .load(Ordering::Relaxed) + == ServerState::Active + }) .count() .to_string(), ])); @@ -248,28 +275,15 @@ where let mut res = BytesMut::new(); res.put(row_description(&columns)); - for (user_pool, pool) in get_all_pools() { - let def = HashMap::default(); - let pool_stats = all_pool_stats - .get(&(user_pool.db.clone(), user_pool.user.clone())) - .unwrap_or(&def); - let pool_config = &pool.settings; + for ((_user_pool, _pool), pool_stats) in all_pool_stats { let mut row = vec![ - user_pool.db.clone(), - user_pool.user.clone(), - pool_config.pool_mode.to_string(), + pool_stats.database(), + pool_stats.user(), + pool_stats.pool_mode().to_string(), ]; - for column in &columns[3..columns.len()] { - let value = match column.0 { - "maxwait" => (pool_stats.get("maxwait_us").unwrap_or(&0) / 1_000_000).to_string(), - "maxwait_us" => { - (pool_stats.get("maxwait_us").unwrap_or(&0) % 1_000_000).to_string() - } - _other_values => pool_stats.get(column.0).unwrap_or(&0).to_string(), - }; - row.push(value); - } + pool_stats.populate_row(&mut row); + pool_stats.clear_maxwait(); res.put(data_row(&row)); } @@ -400,7 +414,7 @@ where for (id, pool) in get_all_pools().iter() { for address in pool.get_addresses_from_host(host) { if !pool.is_banned(&address) { - pool.ban(&address, BanReason::AdminBan(duration_seconds), -1); + pool.ban(&address, BanReason::AdminBan(duration_seconds), None); res.put(data_row(&vec![ id.db.clone(), id.user.clone(), @@ -617,7 +631,6 @@ where ("avg_wait_time", DataType::Numeric), ]; - let all_stats = get_address_stats(); let mut res = BytesMut::new(); res.put(row_description(&columns)); @@ -625,15 +638,10 @@ where for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { let address = pool.address(shard, server); - let stats = match all_stats.get(&address.id) { - Some(stats) => stats.clone(), - None => HashMap::new(), - }; let mut row = vec![address.name(), user_pool.db.clone(), user_pool.user.clone()]; - for column in &columns[3..] { - row.push(stats.get(column.0).unwrap_or(&0).to_string()); - } + let stats = address.stats.clone(); + stats.populate_row(&mut row); res.put(data_row(&row)); } @@ -673,16 +681,16 @@ where for (_, client) in new_map { let row = vec![ - format!("{:#010X}", client.client_id), - client.pool_name, - client.username, - client.application_name.clone(), - client.state.to_string(), - client.transaction_count.to_string(), - client.query_count.to_string(), - client.error_count.to_string(), + format!("{:#010X}", client.client_id()), + client.pool_name(), + client.username(), + client.application_name(), + client.state.load(Ordering::Relaxed).to_string(), + client.transaction_count.load(Ordering::Relaxed).to_string(), + client.query_count.load(Ordering::Relaxed).to_string(), + client.error_count.load(Ordering::Relaxed).to_string(), Instant::now() - .duration_since(client.connect_time) + .duration_since(client.connect_time()) .as_secs() .to_string(), ]; @@ -724,19 +732,20 @@ where res.put(row_description(&columns)); for (_, server) in new_map { + let application_name = server.application_name.read(); let row = vec![ - format!("{:#010X}", server.server_id), - server.pool_name, - server.username, - server.address_name, - server.application_name, - server.state.to_string(), - server.transaction_count.to_string(), - server.query_count.to_string(), - server.bytes_sent.to_string(), - server.bytes_received.to_string(), + format!("{:#010X}", server.server_id()), + server.pool_name(), + server.username(), + server.address_name(), + application_name.clone(), + server.state.load(Ordering::Relaxed).to_string(), + server.transaction_count.load(Ordering::Relaxed).to_string(), + server.query_count.load(Ordering::Relaxed).to_string(), + server.bytes_sent.load(Ordering::Relaxed).to_string(), + server.bytes_received.load(Ordering::Relaxed).to_string(), Instant::now() - .duration_since(server.connect_time) + .duration_since(server.connect_time()) .as_secs() .to_string(), ]; diff --git a/src/client.rs b/src/client.rs index 68f6f5751..f9f4e015d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -3,8 +3,8 @@ use crate::pool::BanReason; /// Handle clients by pretending to be a PostgreSQL server. use bytes::{Buf, BufMut, BytesMut}; use log::{debug, error, info, trace, warn}; - use std::collections::HashMap; +use std::sync::Arc; use std::time::Instant; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; use tokio::net::TcpStream; @@ -19,7 +19,7 @@ use crate::messages::*; use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; -use crate::stats::{get_reporter, Reporter}; +use crate::stats::{ClientStats, PoolStats, ServerStats}; use crate::tls::Tls; use tokio_rustls::server::TlsStream; @@ -66,8 +66,8 @@ pub struct Client { #[allow(dead_code)] parameters: HashMap, - /// Statistics - stats: Reporter, + /// Statistics related to this client + stats: Arc, /// Clients want to talk to admin database. admin: bool, @@ -75,8 +75,8 @@ pub struct Client { /// Last address the client talked to. last_address_id: Option, - /// Last server process id we talked to. - last_server_id: Option, + /// Last server process stats we talked to. + last_server_stats: Option>, /// Connected to server connected_to_server: bool, @@ -135,6 +135,10 @@ pub async fn client_entrypoint( if !client.is_admin() { let _ = drain.send(-1).await; + + if result.is_err() { + client.stats.disconnect(); + } } result @@ -183,6 +187,10 @@ pub async fn client_entrypoint( if !client.is_admin() { let _ = drain.send(-1).await; + + if result.is_err() { + client.stats.disconnect(); + } } result @@ -233,6 +241,10 @@ pub async fn client_entrypoint( if !client.is_admin() { let _ = drain.send(-1).await; + + if result.is_err() { + client.stats.disconnect(); + } } result @@ -258,8 +270,11 @@ pub async fn client_entrypoint( if !client.is_admin() { let _ = drain.send(-1).await; - } + if result.is_err() { + client.stats.disconnect(); + } + } result } @@ -382,7 +397,6 @@ where shutdown: Receiver<()>, admin_only: bool, ) -> Result, Error> { - let stats = get_reporter(); let parameters = parse_startup(bytes.clone())?; // This parameter is mandatory by the protocol. @@ -537,6 +551,25 @@ where ready_for_query(&mut write).await?; trace!("Startup OK"); + let pool_stats = match get_pool(pool_name, username) { + Some(pool) => { + if !admin { + pool.stats + } else { + Arc::new(PoolStats::default()) + } + } + None => Arc::new(PoolStats::default()), + }; + + let stats = Arc::new(ClientStats::new( + process_id, + application_name, + username, + pool_name, + tokio::time::Instant::now(), + pool_stats, + )); Ok(Client { read: BufReader::new(read), @@ -552,7 +585,7 @@ where stats, admin, last_address_id: None, - last_server_id: None, + last_server_stats: None, pool_name: pool_name.clone(), username: username.clone(), application_name: application_name.to_string(), @@ -583,10 +616,10 @@ where secret_key, client_server_map, parameters: HashMap::new(), - stats: get_reporter(), + stats: Arc::new(ClientStats::default()), admin: false, last_address_id: None, - last_server_id: None, + last_server_stats: None, pool_name: String::from("undefined"), username: String::from("undefined"), application_name: String::from("undefined"), @@ -627,12 +660,8 @@ where // The query router determines where the query is going to go, // e.g. primary, replica, which shard. let mut query_router = QueryRouter::new(); - self.stats.client_register( - self.process_id, - self.pool_name.clone(), - self.username.clone(), - self.application_name.clone(), - ); + + self.stats.register(self.stats.clone()); // Our custom protocol loop. // We expect the client to either start a transaction with regular queries @@ -656,6 +685,8 @@ where &mut self.write, "terminating connection due to administrator command" ).await?; + self.stats.disconnect(); + return Ok(()) } @@ -708,6 +739,9 @@ where 'X' => { debug!("Client disconnecting"); + + self.stats.disconnect(); + return Ok(()); } @@ -757,7 +791,7 @@ where current_shard, ), ) - .await?; + .await?; } else { custom_protocol_response_ok(&mut self.write, "SET SHARD").await?; } @@ -802,10 +836,13 @@ where }; debug!("Waiting for connection from pool"); + if !self.admin { + self.stats.waiting(); + } // Grab a server from the pool. let connection = match pool - .get(query_router.shard(), query_router.role(), self.process_id) + .get(query_router.shard(), query_router.role(), &self.stats) .await { Ok(conn) => { @@ -817,6 +854,8 @@ where // but we were unable to grab a connection from the pool // We'll send back an error message and clean the extended // protocol buffer + self.stats.idle(); + if message[0] as char == 'S' { error!("Got Sync message but failed to get a connection from the pool"); self.buffer.clear(); @@ -825,7 +864,7 @@ where .await?; error!("Could not get connection from pool: {{ pool_name: {:?}, username: {:?}, shard: {:?}, role: \"{:?}\", error: \"{:?}\" }}", - self.pool_name.clone(), self.username.clone(), query_router.shard(), query_router.role(), err); + self.pool_name.clone(), self.username.clone(), query_router.shard(), query_router.role(), err); continue; } }; @@ -840,11 +879,10 @@ where self.connected_to_server = true; // Update statistics - self.stats - .client_active(self.process_id, server.server_id()); + self.stats.active(); self.last_address_id = Some(address.id); - self.last_server_id = Some(server.server_id()); + self.last_server_stats = Some(server.stats()); debug!( "Client {:?} talking to server {:?}", @@ -885,6 +923,7 @@ where Ok(Err(err)) => { // Client disconnected inside a transaction. // Clean up the server and re-use it. + self.stats.disconnect(); server.checkin_cleanup().await?; return Err(err); @@ -917,16 +956,26 @@ where 'Q' => { debug!("Sending query to server"); - self.send_and_receive_loop(code, Some(&message), server, &address, &pool) - .await?; + self.send_and_receive_loop( + code, + Some(&message), + server, + &address, + &pool, + &self.stats.clone(), + ) + .await?; if !server.in_transaction() { // Report transaction executed statistics. - self.stats.transaction(self.process_id, server.server_id()); + self.stats.transaction(); + server.stats().transaction(&self.application_name); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. if self.transaction_mode { + self.stats.idle(); + break; } } @@ -935,6 +984,7 @@ where // Terminate 'X' => { server.checkin_cleanup().await?; + self.stats.disconnect(); self.release(); return Ok(()); @@ -987,13 +1037,21 @@ where } } - self.send_and_receive_loop(code, None, server, &address, &pool) - .await?; + self.send_and_receive_loop( + code, + None, + server, + &address, + &pool, + &self.stats.clone(), + ) + .await?; self.buffer.clear(); if !server.in_transaction() { - self.stats.transaction(self.process_id, server.server_id()); + self.stats.transaction(); + server.stats().transaction(&self.application_name); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -1028,7 +1086,9 @@ where // Clear the buffer self.buffer.clear(); - let response = self.receive_server_message(server, &address, &pool).await?; + let response = self + .receive_server_message(server, &address, &pool, &self.stats.clone()) + .await?; match write_all_half(&mut self.write, &response).await { Ok(_) => (), @@ -1039,7 +1099,8 @@ where }; if !server.in_transaction() { - self.stats.transaction(self.process_id, server.server_id()); + self.stats.transaction(); + server.stats().transaction(&self.application_name); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -1060,11 +1121,11 @@ where // The server is no longer bound to us, we can't cancel it's queries anymore. debug!("Releasing server back into the pool"); server.checkin_cleanup().await?; - self.stats.server_idle(server.server_id()); + server.stats().idle(); self.connected_to_server = false; self.release(); - self.stats.client_idle(self.process_id); + self.stats.idle(); } } @@ -1104,6 +1165,7 @@ where server: &mut Server, address: &Address, pool: &ConnectionPool, + client_stats: &ClientStats, ) -> Result<(), Error> { debug!("Sending {} to server", code); @@ -1119,7 +1181,9 @@ where // Read all data the server has to offer, which can be multiple messages // buffered in 8196 bytes chunks. loop { - let response = self.receive_server_message(server, address, pool).await?; + let response = self + .receive_server_message(server, address, pool, client_stats) + .await?; match write_all_half(&mut self.write, &response).await { Ok(_) => (), @@ -1135,10 +1199,10 @@ where } // Report query executed statistics. - self.stats.query( - self.process_id, - server.server_id(), - Instant::now().duration_since(query_start).as_millis(), + client_stats.query(); + server.stats().query( + Instant::now().duration_since(query_start).as_millis() as u64, + &self.application_name, ); Ok(()) @@ -1154,7 +1218,7 @@ where match server.send(message).await { Ok(_) => Ok(()), Err(err) => { - pool.ban(address, BanReason::MessageSendFailed, self.process_id); + pool.ban(address, BanReason::MessageSendFailed, Some(&self.stats)); Err(err) } } @@ -1165,6 +1229,7 @@ where server: &mut Server, address: &Address, pool: &ConnectionPool, + client_stats: &ClientStats, ) -> Result { if pool.settings.user.statement_timeout > 0 { match tokio::time::timeout( @@ -1176,7 +1241,7 @@ where Ok(result) => match result { Ok(message) => Ok(message), Err(err) => { - pool.ban(address, BanReason::MessageReceiveFailed, self.process_id); + pool.ban(address, BanReason::MessageReceiveFailed, Some(client_stats)); error_response_terminal( &mut self.write, &format!("error receiving data from server: {:?}", err), @@ -1191,7 +1256,7 @@ where address, pool.settings.user.username ); server.mark_bad(); - pool.ban(address, BanReason::StatementTimeout, self.process_id); + pool.ban(address, BanReason::StatementTimeout, Some(client_stats)); error_response_terminal(&mut self.write, "pool statement timeout").await?; Err(Error::StatementTimeout) } @@ -1200,7 +1265,7 @@ where match server.recv().await { Ok(message) => Ok(message), Err(err) => { - pool.ban(address, BanReason::MessageReceiveFailed, self.process_id); + pool.ban(address, BanReason::MessageReceiveFailed, Some(client_stats)); error_response_terminal( &mut self.write, &format!("error receiving data from server: {:?}", err), @@ -1220,9 +1285,9 @@ impl Drop for Client { // Dirty shutdown // TODO: refactor, this is not the best way to handle state management. - self.stats.client_disconnecting(self.process_id); - if self.connected_to_server && self.last_server_id.is_some() { - self.stats.server_idle(self.last_server_id.unwrap()); + + if self.connected_to_server && self.last_server_stats.is_some() { + self.last_server_stats.as_ref().unwrap().idle(); } } } diff --git a/src/config.rs b/src/config.rs index a3182e2be..644532ac1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -15,6 +15,7 @@ use tokio::io::AsyncReadExt; use crate::errors::Error; use crate::pool::{ClientServerMap, ConnectionPool}; use crate::sharding::ShardingFunction; +use crate::stats::AddressStats; use crate::tls::{load_certs, load_keys}; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -62,7 +63,7 @@ impl PartialEq for Option { } /// Address identifying a PostgreSQL server uniquely. -#[derive(Clone, PartialEq, Hash, std::cmp::Eq, Debug)] +#[derive(Clone, Debug)] pub struct Address { /// Unique ID per addressable Postgres server. pub id: usize, @@ -96,6 +97,9 @@ pub struct Address { /// List of addresses to receive mirrored traffic. pub mirrors: Vec
, + + /// Address stats + pub stats: Arc, } impl Default for Address { @@ -112,10 +116,46 @@ impl Default for Address { username: String::from("username"), pool_name: String::from("pool_name"), mirrors: Vec::new(), + stats: Arc::new(AddressStats::default()), } } } +// We need to implement PartialEq by ourselves so we skip stats in the comparison +impl PartialEq for Address { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + && self.host == other.host + && self.port == other.port + && self.shard == other.shard + && self.address_index == other.address_index + && self.replica_number == other.replica_number + && self.database == other.database + && self.role == other.role + && self.username == other.username + && self.pool_name == other.pool_name + && self.mirrors == other.mirrors + } +} +impl Eq for Address {} + +// We need to implement Hash by ourselves so we skip stats in the comparison +impl Hash for Address { + fn hash(&self, state: &mut H) { + self.id.hash(state); + self.host.hash(state); + self.port.hash(state); + self.shard.hash(state); + self.address_index.hash(state); + self.replica_number.hash(state); + self.database.hash(state); + self.role.hash(state); + self.username.hash(state); + self.pool_name.hash(state); + self.mirrors.hash(state); + } +} + impl Address { /// Address name (aka database) used in `SHOW STATS`, `SHOW DATABASES`, and `SHOW POOLS`. pub fn name(&self) -> String { diff --git a/src/main.rs b/src/main.rs index e2ff5d8d8..a59da2108 100644 --- a/src/main.rs +++ b/src/main.rs @@ -162,8 +162,7 @@ fn main() -> Result<(), Box> { let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); // Statistics reporting. - let (stats_tx, stats_rx) = mpsc::channel(500_000); - REPORTER.store(Arc::new(Reporter::new(stats_tx.clone()))); + REPORTER.store(Arc::new(Reporter::default())); // Connection pool that allows to query all shards and replicas. match ConnectionPool::from_config(client_server_map.clone()).await { @@ -175,7 +174,7 @@ fn main() -> Result<(), Box> { }; tokio::task::spawn(async move { - let mut stats_collector = Collector::new(stats_rx, stats_tx.clone()); + let mut stats_collector = Collector::default(); stats_collector.collect().await; }); diff --git a/src/mirrors.rs b/src/mirrors.rs index ab2b2dc40..128fe220c 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -1,11 +1,13 @@ +use std::sync::Arc; + /// A mirrored PostgreSQL client. /// Packets arrive to us through a channel from the main client and we send them to the server. use bb8::Pool; use bytes::{Bytes, BytesMut}; use crate::config::{get_config, Address, Role, User}; -use crate::pool::{ClientServerMap, ServerPool}; -use crate::stats::get_reporter; +use crate::pool::{ClientServerMap, PoolIdentifier, ServerPool}; +use crate::stats::PoolStats; use log::{error, info, trace, warn}; use tokio::sync::mpsc::{channel, Receiver, Sender}; @@ -21,20 +23,24 @@ impl MirroredClient { async fn create_pool(&self) -> Pool { let config = get_config(); let default = std::time::Duration::from_millis(10_000).as_millis() as u64; - let (connection_timeout, idle_timeout) = match config.pools.get(&self.address.pool_name) { - Some(cfg) => ( - cfg.connect_timeout.unwrap_or(default), - cfg.idle_timeout.unwrap_or(default), - ), - None => (default, default), - }; + let (connection_timeout, idle_timeout, cfg) = + match config.pools.get(&self.address.pool_name) { + Some(cfg) => ( + cfg.connect_timeout.unwrap_or(default), + cfg.idle_timeout.unwrap_or(default), + cfg.clone(), + ), + None => (default, default, crate::config::Pool::default()), + }; + + let identifier = PoolIdentifier::new(&self.database, &self.user.username); let manager = ServerPool::new( self.address.clone(), self.user.clone(), self.database.as_str(), ClientServerMap::default(), - get_reporter(), + Arc::new(PoolStats::new(identifier, cfg.clone())), ); Pool::builder() diff --git a/src/pool.rs b/src/pool.rs index 3a6ec3e66..f6f9118b4 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -22,7 +22,7 @@ use crate::errors::Error; use crate::server::Server; use crate::sharding::ShardingFunction; -use crate::stats::{get_reporter, Reporter}; +use crate::stats::{AddressStats, ClientStats, PoolStats, ServerStats}; pub type ProcessId = i32; pub type SecretKey = i32; @@ -51,7 +51,7 @@ pub enum BanReason { /// An identifier for a PgCat pool, /// a database visible to clients. -#[derive(Hash, Debug, Clone, PartialEq, Eq)] +#[derive(Hash, Debug, Clone, PartialEq, Eq, Default)] pub struct PoolIdentifier { // The name of the database clients want to connect to. pub db: String, @@ -161,10 +161,6 @@ pub struct ConnectionPool { /// that should not be queried. banlist: BanList, - /// The statistics aggregator runs in a separate task - /// and receives stats from clients, servers, and the pool. - stats: Reporter, - /// The server information (K messages) have to be passed to the /// clients on startup. We pre-connect to all shards and replicas /// on pool creation and save the K messages here. @@ -185,6 +181,8 @@ pub struct ConnectionPool { /// If the pool has been paused or not. paused: Arc, paused_waiter: Arc, + + pub stats: Arc, } impl ConnectionPool { @@ -201,6 +199,7 @@ impl ConnectionPool { // There is one pool per database/user pair. for user in pool_config.users.values() { let old_pool_ref = get_pool(pool_name, &user.username); + let identifier = PoolIdentifier::new(pool_name, &user.username); match old_pool_ref { Some(pool) => { @@ -211,10 +210,7 @@ impl ConnectionPool { "[pool: {}][user: {}] has not changed", pool_name, user.username ); - new_pools.insert( - PoolIdentifier::new(pool_name, &user.username), - pool.clone(), - ); + new_pools.insert(identifier.clone(), pool.clone()); continue; } } @@ -234,6 +230,10 @@ impl ConnectionPool { .clone() .into_keys() .collect::>(); + let pool_stats = Arc::new(PoolStats::new(identifier, pool_config.clone())); + + // Allow the pool to be seen in statistics + pool_stats.register(pool_stats.clone()); // Sort by shard number to ensure consistency. shard_ids.sort_by_key(|k| k.parse::().unwrap()); @@ -266,6 +266,7 @@ impl ConnectionPool { username: user.username.clone(), pool_name: pool_name.clone(), mirrors: vec![], + stats: Arc::new(AddressStats::default()), }); address_id += 1; } @@ -283,6 +284,7 @@ impl ConnectionPool { username: user.username.clone(), pool_name: pool_name.clone(), mirrors: mirror_addresses, + stats: Arc::new(AddressStats::default()), }; address_id += 1; @@ -296,7 +298,7 @@ impl ConnectionPool { user.clone(), &shard.database, client_server_map.clone(), - get_reporter(), + pool_stats.clone(), ); let connect_timeout = match pool_config.connect_timeout { @@ -331,9 +333,9 @@ impl ConnectionPool { let pool = ConnectionPool { databases: shards, + stats: pool_stats, addresses, banlist: Arc::new(RwLock::new(banlist)), - stats: get_reporter(), config_hash: new_pool_hash_value, server_info: Arc::new(RwLock::new(BytesMut::new())), settings: PoolSettings { @@ -476,9 +478,9 @@ impl ConnectionPool { /// Get a connection from the pool. pub async fn get( &self, - shard: usize, // shard number - role: Option, // primary or replica - client_process_id: i32, // client id + shard: usize, // shard number + role: Option, // primary or replica + client_stats: &ClientStats, // client id ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { let mut candidates: Vec<&Address> = self.addresses[shard] .iter() @@ -517,7 +519,7 @@ impl ConnectionPool { // Indicate we're waiting on a server connection from a pool. let now = Instant::now(); - self.stats.client_waiting(client_process_id); + client_stats.waiting(); // Check if we can connect let mut conn = match self.databases[address.shard][address.address_index] @@ -527,9 +529,10 @@ impl ConnectionPool { Ok(conn) => conn, Err(err) => { error!("Banning instance {:?}, error: {:?}", address, err); - self.ban(address, BanReason::FailedCheckout, client_process_id); - self.stats - .client_checkout_error(client_process_id, address.id); + self.ban(address, BanReason::FailedCheckout, Some(client_stats)); + address.stats.error(); + client_stats.idle(); + client_stats.checkout_error(); continue; } }; @@ -546,18 +549,18 @@ impl ConnectionPool { // since we last checked the server is ok. // Health checks are pretty expensive. if !require_healthcheck { - self.stats.checkout_time( - now.elapsed().as_micros(), - client_process_id, - server.server_id(), - ); - self.stats - .server_active(client_process_id, server.server_id()); + let checkout_time: u64 = now.elapsed().as_micros() as u64; + client_stats.checkout_time(checkout_time); + server + .stats() + .checkout_time(checkout_time, client_stats.application_name()); + server.stats().active(client_stats.application_name()); + return Ok((conn, address.clone())); } if self - .run_health_check(address, server, now, client_process_id) + .run_health_check(address, server, now, client_stats) .await { return Ok((conn, address.clone())); @@ -565,7 +568,6 @@ impl ConnectionPool { continue; } } - Err(Error::AllServersDown) } @@ -574,11 +576,11 @@ impl ConnectionPool { address: &Address, server: &mut Server, start: Instant, - client_process_id: i32, + client_info: &ClientStats, ) -> bool { debug!("Running health check on server {:?}", address); - self.stats.server_tested(server.server_id()); + server.stats().tested(); match tokio::time::timeout( tokio::time::Duration::from_millis(self.settings.healthcheck_timeout), @@ -589,13 +591,13 @@ impl ConnectionPool { // Check if health check succeeded. Ok(res) => match res { Ok(_) => { - self.stats.checkout_time( - start.elapsed().as_micros(), - client_process_id, - server.server_id(), - ); - self.stats - .server_active(client_process_id, server.server_id()); + let checkout_time: u64 = start.elapsed().as_micros() as u64; + client_info.checkout_time(checkout_time); + server + .stats() + .checkout_time(checkout_time, client_info.application_name()); + server.stats().active(client_info.application_name()); + return true; } @@ -620,14 +622,14 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(&address, BanReason::FailedHealthCheck, client_process_id); + self.ban(&address, BanReason::FailedHealthCheck, Some(client_info)); return false; } /// Ban an address (i.e. replica). It no longer will serve /// traffic for any new transactions. Existing transactions on that replica /// will finish successfully or error out to the clients. - pub fn ban(&self, address: &Address, reason: BanReason, client_id: i32) { + pub fn ban(&self, address: &Address, reason: BanReason, client_info: Option<&ClientStats>) { // Primary can never be banned if address.role == Role::Primary { return; @@ -636,7 +638,10 @@ impl ConnectionPool { let now = chrono::offset::Utc::now().naive_utc(); let mut guard = self.banlist.write(); error!("Banning {:?}", address); - self.stats.client_ban_error(client_id, address.id); + if let Some(client_info) = client_info { + client_info.ban_error(); + address.stats.error(); + } guard[address.shard].insert(address.clone(), (reason, now)); } @@ -797,7 +802,7 @@ pub struct ServerPool { user: User, database: String, client_server_map: ClientServerMap, - stats: Reporter, + stats: Arc, } impl ServerPool { @@ -806,11 +811,11 @@ impl ServerPool { user: User, database: &str, client_server_map: ClientServerMap, - stats: Reporter, + stats: Arc, ) -> ServerPool { ServerPool { address, - user, + user: user.clone(), database: database.to_string(), client_server_map, stats, @@ -826,34 +831,31 @@ impl ManageConnection for ServerPool { /// Attempts to create a new connection. async fn connect(&self) -> Result { info!("Creating a new server connection {:?}", self.address); - let server_id = rand::random::(); - self.stats.server_register( - server_id, - self.address.id, - self.address.name(), - self.address.pool_name.clone(), - self.address.username.clone(), - ); - self.stats.server_login(server_id); + let stats = Arc::new(ServerStats::new( + self.address.clone(), + self.stats.clone(), + tokio::time::Instant::now(), + )); + + stats.register(stats.clone()); // Connect to the PostgreSQL server. match Server::startup( - server_id, &self.address, &self.user, &self.database, self.client_server_map.clone(), - self.stats.clone(), + stats.clone(), ) .await { Ok(conn) => { - self.stats.server_idle(server_id); + stats.idle(); Ok(conn) } Err(err) => { - self.stats.server_disconnecting(server_id); + stats.disconnect(); Err(err) } } @@ -881,11 +883,3 @@ pub fn get_pool(db: &str, user: &str) -> Option { pub fn get_all_pools() -> HashMap { (*(*POOLS.load())).clone() } - -/// How many total servers we have in the config. -pub fn get_number_of_addresses() -> usize { - get_all_pools() - .iter() - .map(|(_, pool)| pool.databases()) - .sum() -} diff --git a/src/prometheus.rs b/src/prometheus.rs index e596f9f99..6e578bf07 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -5,10 +5,12 @@ use phf::phf_map; use std::collections::HashMap; use std::fmt; use std::net::SocketAddr; +use std::sync::atomic::Ordering; +use std::sync::Arc; use crate::config::Address; use crate::pool::get_all_pools; -use crate::stats::{get_address_stats, get_pool_stats, get_server_stats, ServerInformation}; +use crate::stats::{get_pool_stats, get_server_stats, ServerStats}; struct MetricHelpType { help: &'static str, @@ -220,7 +222,7 @@ impl PrometheusMetric { Self::from_name(&format!("servers_{}", name), value, labels) } - fn from_address(address: &Address, name: &str, value: i64) -> Option> { + fn from_address(address: &Address, name: &str, value: u64) -> Option> { let mut labels = HashMap::new(); labels.insert("host", address.host.clone()); labels.insert("shard", address.shard.to_string()); @@ -231,7 +233,7 @@ impl PrometheusMetric { Self::from_name(&format!("stats_{}", name), value, labels) } - fn from_pool(pool: &(String, String), name: &str, value: i64) -> Option> { + fn from_pool(pool: &(String, String), name: &str, value: u64) -> Option> { let mut labels = HashMap::new(); labels.insert("pool", pool.0.clone()); labels.insert("user", pool.1.clone()); @@ -261,20 +263,18 @@ async fn prometheus_stats(request: Request) -> Result, hype // Adds metrics shown in a SHOW STATS admin command. fn push_address_stats(lines: &mut Vec) { - let address_stats: HashMap> = get_address_stats(); for (_, pool) in get_all_pools() { for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { let address = pool.address(shard, server); - if let Some(address_stats) = address_stats.get(&address.id) { - for (key, value) in address_stats.iter() { - if let Some(prometheus_metric) = - PrometheusMetric::::from_address(address, key, *value) - { - lines.push(prometheus_metric.to_string()); - } else { - warn!("Metric {} not implemented for {}", key, address.name()); - } + let stats = &*address.stats; + for (key, value) in stats.clone() { + if let Some(prometheus_metric) = + PrometheusMetric::::from_address(address, &key, value) + { + lines.push(prometheus_metric.to_string()); + } else { + warn!("Metric {} not implemented for {}", key, address.name()); } } } @@ -286,8 +286,9 @@ fn push_address_stats(lines: &mut Vec) { fn push_pool_stats(lines: &mut Vec) { let pool_stats = get_pool_stats(); for (pool, stats) in pool_stats.iter() { - for (name, value) in stats.iter() { - if let Some(prometheus_metric) = PrometheusMetric::::from_pool(pool, name, *value) + let stats = &**stats; + for (name, value) in stats.clone() { + if let Some(prometheus_metric) = PrometheusMetric::::from_pool(pool, &name, value) { lines.push(prometheus_metric.to_string()); } else { @@ -330,9 +331,9 @@ fn push_database_stats(lines: &mut Vec) { // Adds relevant metrics shown in a SHOW SERVERS admin command. fn push_server_stats(lines: &mut Vec) { let server_stats = get_server_stats(); - let mut server_stats_by_addresses = HashMap::::new(); - for (_, info) in server_stats { - server_stats_by_addresses.insert(info.address_name.clone(), info); + let mut server_stats_by_addresses = HashMap::>::new(); + for (_, stats) in server_stats { + server_stats_by_addresses.insert(stats.address_name(), stats); } for (_, pool) in get_all_pools() { @@ -341,11 +342,23 @@ fn push_server_stats(lines: &mut Vec) { let address = pool.address(shard, server); if let Some(server_info) = server_stats_by_addresses.get(&address.name()) { let metrics = [ - ("bytes_received", server_info.bytes_received), - ("bytes_sent", server_info.bytes_sent), - ("transaction_count", server_info.transaction_count), - ("query_count", server_info.query_count), - ("error_count", server_info.error_count), + ( + "bytes_received", + server_info.bytes_received.load(Ordering::Relaxed), + ), + ("bytes_sent", server_info.bytes_sent.load(Ordering::Relaxed)), + ( + "transaction_count", + server_info.transaction_count.load(Ordering::Relaxed), + ), + ( + "query_count", + server_info.query_count.load(Ordering::Relaxed), + ), + ( + "error_count", + server_info.error_count.load(Ordering::Relaxed), + ), ]; for (key, value) in metrics { if let Some(prometheus_metric) = diff --git a/src/server.rs b/src/server.rs index b3dbd6f7f..d09313ec8 100644 --- a/src/server.rs +++ b/src/server.rs @@ -3,6 +3,7 @@ use bytes::{Buf, BufMut, BytesMut}; use log::{debug, error, info, trace, warn}; use std::io::Read; +use std::sync::Arc; use std::time::SystemTime; use tokio::io::{AsyncReadExt, BufReader}; use tokio::net::{ @@ -17,12 +18,10 @@ use crate::messages::*; use crate::mirrors::MirroringManager; use crate::pool::ClientServerMap; use crate::scram::ScramSha256; -use crate::stats::Reporter; +use crate::stats::ServerStats; /// Server state. pub struct Server { - server_id: i32, - /// Server host, e.g. localhost, /// port, e.g. 5432, and role, e.g. primary or replica. address: Address, @@ -62,7 +61,7 @@ pub struct Server { connected_at: chrono::naive::NaiveDateTime, /// Reports various metrics, e.g. data sent & received. - stats: Reporter, + stats: Arc, /// Application name using the server at the moment. application_name: String, @@ -77,12 +76,11 @@ impl Server { /// Pretend to be the Postgres client and connect to the server given host, port and credentials. /// Perform the authentication and return the server in a ready for query state. pub async fn startup( - server_id: i32, address: &Address, user: &User, database: &str, client_server_map: ClientServerMap, - stats: Reporter, + stats: Arc, ) -> Result { let mut stream = match TcpStream::connect(&format!("{}:{}", &address.host, address.port)).await { @@ -325,7 +323,6 @@ impl Server { write, buffer: BytesMut::with_capacity(8196), server_info, - server_id, process_id, secret_key, in_transaction: false, @@ -396,7 +393,7 @@ impl Server { /// Send messages to the server from the client. pub async fn send(&mut self, messages: &BytesMut) -> Result<(), Error> { self.mirror_send(messages); - self.stats.data_sent(messages.len(), self.server_id); + self.stats().data_sent(messages.len()); match write_all_half(&mut self.write, messages).await { Ok(_) => { @@ -545,7 +542,7 @@ impl Server { let bytes = self.buffer.clone(); // Keep track of how much data we got from the server for stats. - self.stats.data_received(bytes.len(), self.server_id); + self.stats().data_received(bytes.len()); // Clear the buffer for next query. self.buffer.clear(); @@ -665,18 +662,17 @@ impl Server { } } + /// get Server stats + pub fn stats(&self) -> Arc { + self.stats.clone() + } + /// Get the servers address. #[allow(dead_code)] pub fn address(&self) -> Address { self.address.clone() } - /// Get the server connection identifier - /// Used to uniquely identify connection in statistics - pub fn server_id(&self) -> i32 { - self.server_id - } - // Get server's latest response timestamp pub fn last_activity(&self) -> SystemTime { self.last_activity @@ -708,7 +704,9 @@ impl Drop for Server { /// for a write. fn drop(&mut self) { self.mirror_disconnect(); - self.stats.server_disconnecting(self.server_id); + + // Update statistics + self.stats.disconnect(); let mut bytes = BytesMut::with_capacity(4); bytes.put_u8(b'X'); diff --git a/src/stats.rs b/src/stats.rs index 3f7e9d618..5b7895b4e 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,43 +1,46 @@ -use arc_swap::ArcSwap; +use crate::pool::PoolIdentifier; /// Statistics and reporting. -use log::{error, info, trace, warn}; +use arc_swap::ArcSwap; + +use log::{info, warn}; use once_cell::sync::Lazy; +use parking_lot::RwLock; use std::collections::HashMap; + use std::sync::Arc; -use tokio::sync::mpsc::error::TrySendError; -use tokio::sync::mpsc::{channel, Receiver, Sender}; -use tokio::time::Instant; -use crate::pool::{get_all_pools, get_number_of_addresses}; +// Structs that hold stats for different resources +pub mod address; +pub mod client; +pub mod pool; +pub mod server; +pub use address::AddressStats; +pub use client::{ClientState, ClientStats}; +pub use pool::PoolStats; +pub use server::{ServerState, ServerStats}; /// Convenience types for various stats -type ClientStatesLookup = HashMap; -type ServerStatesLookup = HashMap; -type PoolStatsLookup = HashMap<(String, String), HashMap>; -type AddressStatsLookup = HashMap>; +type ClientStatesLookup = HashMap>; +type ServerStatesLookup = HashMap>; +type PoolStatsLookup = HashMap<(String, String), Arc>; -/// Stats for individual client connections updated every second +/// Stats for individual client connections /// Used in SHOW CLIENTS. -static LATEST_CLIENT_STATS: Lazy> = - Lazy::new(|| ArcSwap::from_pointee(ClientStatesLookup::default())); +static CLIENT_STATS: Lazy>> = + Lazy::new(|| Arc::new(RwLock::new(ClientStatesLookup::default()))); -/// Stats for individual server connections updated every second +/// Stats for individual server connections /// Used in SHOW SERVERS. -static LATEST_SERVER_STATS: Lazy> = - Lazy::new(|| ArcSwap::from_pointee(ServerStatesLookup::default())); +static SERVER_STATS: Lazy>> = + Lazy::new(|| Arc::new(RwLock::new(ServerStatesLookup::default()))); -/// Aggregate stats for each pool (a pool is identified by database name and username) updated every second +/// Aggregate stats for each pool (a pool is identified by database name and username) /// Used in SHOW POOLS. -static LATEST_POOL_STATS: Lazy> = - Lazy::new(|| ArcSwap::from_pointee(PoolStatsLookup::default())); - -/// Aggregate stats for individual database instances, updated every second, averages are calculated every 15 -/// Used in SHOW STATS. -static LATEST_ADDRESS_STATS: Lazy> = - Lazy::new(|| ArcSwap::from_pointee(AddressStatsLookup::default())); +static POOL_STATS: Lazy>> = + Lazy::new(|| Arc::new(RwLock::new(PoolStatsLookup::default()))); /// The statistics reporter. An instance is given to each possible source of statistics, -/// e.g. clients, servers, connection pool. +/// e.g. client stats, server stats, connection pool stats. pub static REPORTER: Lazy> = Lazy::new(|| ArcSwap::from_pointee(Reporter::default())); @@ -45,989 +48,89 @@ pub static REPORTER: Lazy> = /// 15 seconds. static STAT_PERIOD: u64 = 15000; -/// The various states that a client can be in -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum ClientState { - Idle, - Waiting, - Active, -} -impl std::fmt::Display for ClientState { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match *self { - ClientState::Idle => write!(f, "idle"), - ClientState::Waiting => write!(f, "waiting"), - ClientState::Active => write!(f, "active"), - } - } -} - -/// The various states that a server can be in -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum ServerState { - Login, - Active, - Tested, - Idle, -} -impl std::fmt::Display for ServerState { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match *self { - ServerState::Login => write!(f, "login"), - ServerState::Active => write!(f, "active"), - ServerState::Tested => write!(f, "tested"), - ServerState::Idle => write!(f, "idle"), - } - } -} - -/// Information we keep track off which can be queried by SHOW CLIENTS -#[derive(Debug, Clone)] -pub struct ClientInformation { - pub state: ClientState, - pub connect_time: Instant, - - /// A random integer assigned to the client and used by stats to track the client - pub client_id: i32, - - pub application_name: String, - pub username: String, - pub pool_name: String, - - /// Total time spent waiting for a connection from pool, measures in microseconds - pub total_wait_time: u64, - - pub transaction_count: u64, - pub query_count: u64, - pub error_count: u64, -} - -/// Information we keep track off which can be queried by SHOW SERVERS -#[derive(Debug, Clone)] -pub struct ServerInformation { - pub state: ServerState, - pub connect_time: Instant, - - /// A random integer assigned to the server and used by stats to track the server - pub server_id: i32, - - pub address_name: String, - pub address_id: usize, - - pub username: String, - pub pool_name: String, - pub application_name: String, - - pub bytes_sent: u64, - pub bytes_received: u64, - - pub transaction_count: u64, - pub query_count: u64, - pub error_count: u64, -} - -/// The names for the events reported -/// to the statistics collector. -#[derive(Debug, Clone)] -enum EventName { - CheckoutTime { - client_id: i32, - server_id: i32, - }, - Query { - client_id: i32, - server_id: i32, - duration_ms: u128, - }, - Transaction { - client_id: i32, - server_id: i32, - }, - - DataSentToServer { - server_id: i32, - }, - DataReceivedFromServer { - server_id: i32, - }, - - ClientRegistered { - client_id: i32, - pool_name: String, - username: String, - application_name: String, - }, - ClientIdle { - client_id: i32, - }, - ClientWaiting { - client_id: i32, - }, - ClientActive { - client_id: i32, - #[allow(dead_code)] - server_id: i32, - }, - ClientDisconnecting { - client_id: i32, - }, - ClientCheckoutError { - client_id: i32, - #[allow(dead_code)] - address_id: usize, - }, - ClientBanError { - client_id: i32, - #[allow(dead_code)] - address_id: usize, - }, - - ServerRegistered { - server_id: i32, - address_id: usize, - address_name: String, - pool_name: String, - username: String, - }, - ServerLogin { - server_id: i32, - }, - ServerIdle { - server_id: i32, - }, - ServerTested { - server_id: i32, - }, - ServerActive { - #[allow(dead_code)] - client_id: i32, - server_id: i32, - }, - ServerDisconnecting { - server_id: i32, - }, - - UpdateStats { - pool_name: String, - username: String, - }, - UpdateAverages { - address_id: usize, - }, -} - -/// Event data sent to the collector -/// from clients and servers. -#[derive(Debug, Clone)] -pub struct Event { - /// The name of the event being reported. - name: EventName, - - /// The value being reported. Meaning differs based on event name. - value: i64, -} - /// The statistics reporter. An instance is given /// to each possible source of statistics, /// e.g. clients, servers, connection pool. -#[derive(Clone, Debug)] -pub struct Reporter { - tx: Sender, -} - -impl Default for Reporter { - fn default() -> Reporter { - let (tx, _rx) = channel(5); - Reporter { tx } - } -} +#[derive(Clone, Debug, Default)] +pub struct Reporter {} impl Reporter { - /// Create a new Reporter instance. - pub fn new(tx: Sender) -> Reporter { - Reporter { tx } - } - - /// Send statistics to the task keeping track of stats. - fn send(&self, event: Event) { - let name = event.name.clone(); - let result = self.tx.try_send(event.clone()); - - match result { - Ok(_) => trace!( - "{:?} event reported successfully, capacity: {} {:?}", - name, - self.tx.capacity(), - event - ), - - Err(err) => match err { - TrySendError::Full { .. } => error!("{:?} event dropped, buffer full", name), - TrySendError::Closed { .. } => error!("{:?} event dropped, channel closed", name), - }, - }; - } - - /// Report a query executed by a client against a server - pub fn query(&self, client_id: i32, server_id: i32, duration_ms: u128) { - let event = Event { - name: EventName::Query { - client_id, - server_id, - duration_ms, - }, - value: 1, - }; - self.send(event); - } - - /// Report a transaction executed by a client a server - /// we report each individual queries outside a transaction as a transaction - /// We only count the initial BEGIN as a transaction, all queries within do not - /// count as transactions - pub fn transaction(&self, client_id: i32, server_id: i32) { - let event = Event { - name: EventName::Transaction { - client_id, - server_id, - }, - value: 1, - }; - self.send(event); - } - - /// Report data sent to a server - pub fn data_sent(&self, amount_bytes: usize, server_id: i32) { - let event = Event { - name: EventName::DataSentToServer { server_id }, - value: amount_bytes as i64, - }; - self.send(event) - } - - /// Report data received from a server - pub fn data_received(&self, amount_bytes: usize, server_id: i32) { - let event = Event { - name: EventName::DataReceivedFromServer { server_id }, - value: amount_bytes as i64, - }; - self.send(event) - } - - /// Reportes the time spent by a client waiting to get a healthy connection from the pool - pub fn checkout_time(&self, microseconds: u128, client_id: i32, server_id: i32) { - let event = Event { - name: EventName::CheckoutTime { - client_id, - server_id, - }, - value: microseconds as i64, - }; - self.send(event) - } - /// Register a client with the stats system. The stats system uses client_id /// to track and aggregate statistics from all source that relate to that client - pub fn client_register( - &self, - client_id: i32, - pool_name: String, - username: String, - app_name: String, - ) { - let event = Event { - name: EventName::ClientRegistered { - client_id, - pool_name, - username, - application_name: app_name, - }, - value: 1, - }; - self.send(event); - } - - /// Reports a client is waiting for a connection - pub fn client_waiting(&self, client_id: i32) { - let event = Event { - name: EventName::ClientWaiting { client_id }, - value: 1, - }; - self.send(event) - } - - /// Reports a client has had the server assigned to it be banned - pub fn client_ban_error(&self, client_id: i32, address_id: usize) { - let event = Event { - name: EventName::ClientBanError { - client_id, - address_id, - }, - value: 1, - }; - self.send(event) - } - - /// Reports a client has failed to obtain a connection from a connection pool - pub fn client_checkout_error(&self, client_id: i32, address_id: usize) { - let event = Event { - name: EventName::ClientCheckoutError { - client_id, - address_id, - }, - value: 1, - }; - self.send(event) - } - - /// Reports a client is done waiting for a connection and is about to query the server. - pub fn client_active(&self, client_id: i32, server_id: i32) { - let event = Event { - name: EventName::ClientActive { - client_id, - server_id, - }, - value: 1, - }; - self.send(event) - } + fn client_register(&self, client_id: i32, stats: Arc) { + if CLIENT_STATS.read().get(&client_id).is_some() { + warn!("Client {:?} was double registered!", client_id); + return; + } - /// Reports a client is done querying the server and is no longer assigned a server connection - pub fn client_idle(&self, client_id: i32) { - let event = Event { - name: EventName::ClientIdle { client_id }, - value: 1, - }; - self.send(event) + CLIENT_STATS.write().insert(client_id, stats); } /// Reports a client is disconecting from the pooler. - pub fn client_disconnecting(&self, client_id: i32) { - let event = Event { - name: EventName::ClientDisconnecting { client_id }, - value: 1, - }; - self.send(event) + fn client_disconnecting(&self, client_id: i32) { + CLIENT_STATS.write().remove(&client_id); } /// Register a server connection with the stats system. The stats system uses server_id /// to track and aggregate statistics from all source that relate to that server - pub fn server_register( - &self, - server_id: i32, - address_id: usize, - address_name: String, - pool_name: String, - username: String, - ) { - let event = Event { - name: EventName::ServerRegistered { - server_id, - address_id, - address_name, - pool_name, - username, - }, - value: 1, - }; - self.send(event); - } - - /// Reports a server connection has been assigned to a client that - /// is about to query the server - pub fn server_active(&self, client_id: i32, server_id: i32) { - let event = Event { - name: EventName::ServerActive { - client_id, - server_id, - }, - value: 1, - }; - self.send(event) - } - - /// Reports a server connection is no longer assigned to a client - /// and is available for the next client to pick it up - pub fn server_idle(&self, server_id: i32) { - let event = Event { - name: EventName::ServerIdle { server_id }, - value: 1, - }; - self.send(event) - } - - /// Reports a server connection is attempting to login. - pub fn server_login(&self, server_id: i32) { - let event = Event { - name: EventName::ServerLogin { server_id }, - value: 1, - }; - self.send(event) + fn server_register(&self, server_id: i32, stats: Arc) { + SERVER_STATS.write().insert(server_id, stats); } - - /// Reports a server connection is being tested before being given to a client. - pub fn server_tested(&self, server_id: i32) { - let event = Event { - name: EventName::ServerTested { server_id }, - value: 1, - }; - - self.send(event) + /// Reports a server connection is disconecting from the pooler. + fn server_disconnecting(&self, server_id: i32) { + SERVER_STATS.write().remove(&server_id); } - /// Reports a server connection is disconecting from the pooler. - pub fn server_disconnecting(&self, server_id: i32) { - let event = Event { - name: EventName::ServerDisconnecting { server_id }, - value: 1, - }; - self.send(event) + /// Register a pool with the stats system. + fn pool_register(&self, identifier: PoolIdentifier, stats: Arc) { + POOL_STATS + .write() + .insert((identifier.db, identifier.user), stats); } } -/// The statistics collector which is receiving statistics -/// from clients, servers, and the connection pool. There is -/// only one collector (kind of like a singleton). -/// The collector can trigger events on its own, e.g. -/// it updates aggregates every second and averages every -/// 15 seconds. -pub struct Collector { - rx: Receiver, - tx: Sender, -} +/// The statistics collector which used for calculating averages +/// There is only one collector (kind of like a singleton) +/// it updates averages every 15 seconds. +#[derive(Default)] +pub struct Collector {} impl Collector { - /// Create a new collector instance. There should only be one instance - /// at a time. This is ensured by mpsc which allows only one receiver. - pub fn new(rx: Receiver, tx: Sender) -> Collector { - Collector { rx, tx } - } - /// The statistics collection handler. It will collect statistics /// for `address_id`s starting at 0 up to `addresses`. pub async fn collect(&mut self) { info!("Events reporter started"); - let mut client_states = ClientStatesLookup::default(); - let mut server_states = ServerStatesLookup::default(); - let mut pool_stat_lookup = PoolStatsLookup::default(); - - let mut address_stat_lookup = AddressStatsLookup::default(); - let mut address_old_stat_lookup = AddressStatsLookup::default(); - - let tx = self.tx.clone(); - tokio::task::spawn(async move { - let mut interval = - tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD / 15)); - loop { - interval.tick().await; - for (user_pool, _) in get_all_pools() { - let _ = tx.try_send(Event { - name: EventName::UpdateStats { - pool_name: user_pool.db, - username: user_pool.user, - }, - value: 0, - }); - } - } - }); - - let tx = self.tx.clone(); tokio::task::spawn(async move { let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(STAT_PERIOD)); loop { interval.tick().await; - for address_id in 0..get_number_of_addresses() { - let _ = tx.try_send(Event { - name: EventName::UpdateAverages { address_id }, - value: 0, - }); + + for stats in SERVER_STATS.read().values() { + stats.address_stats().update_averages(); } } }); - - // The collector loop - loop { - let stat = match self.rx.recv().await { - Some(stat) => stat, - None => { - info!("Events collector is shutting down"); - return; - } - }; - - // Some are counters, some are gauges... - match stat.name { - EventName::Query { - client_id, - server_id, - duration_ms, - } => { - // Update client stats - let app_name = match client_states.get_mut(&client_id) { - Some(client_info) => { - client_info.query_count += stat.value as u64; - client_info.application_name.to_string() - } - None => String::from("Undefined"), - }; - - // Update server stats and pool aggergation stats - match server_states.get_mut(&server_id) { - Some(server_info) => { - server_info.query_count += stat.value as u64; - server_info.application_name = app_name; - - let address_stats = address_stat_lookup - .entry(server_info.address_id) - .or_insert_with(HashMap::default); - let counter = address_stats - .entry("total_query_count".to_string()) - .or_insert(0); - *counter += stat.value; - - let duration = address_stats - .entry("total_query_time".to_string()) - .or_insert(0); - *duration += duration_ms as i64; - } - None => (), - } - } - - EventName::Transaction { - client_id, - server_id, - } => { - // Update client stats - let app_name = match client_states.get_mut(&client_id) { - Some(client_info) => { - client_info.transaction_count += stat.value as u64; - client_info.application_name.to_string() - } - None => String::from("Undefined"), - }; - - // Update server stats and pool aggergation stats - match server_states.get_mut(&server_id) { - Some(server_info) => { - server_info.transaction_count += stat.value as u64; - server_info.application_name = app_name; - - let address_stats = address_stat_lookup - .entry(server_info.address_id) - .or_insert_with(HashMap::default); - let counter = address_stats - .entry("total_xact_count".to_string()) - .or_insert(0); - *counter += stat.value; - } - None => (), - } - } - - EventName::DataSentToServer { server_id } => { - // Update server stats and address aggergation stats - match server_states.get_mut(&server_id) { - Some(server_info) => { - server_info.bytes_sent += stat.value as u64; - - let address_stats = address_stat_lookup - .entry(server_info.address_id) - .or_insert_with(HashMap::default); - let counter = - address_stats.entry("total_sent".to_string()).or_insert(0); - *counter += stat.value; - } - None => (), - } - } - - EventName::DataReceivedFromServer { server_id } => { - // Update server states and address aggergation stats - match server_states.get_mut(&server_id) { - Some(server_info) => { - server_info.bytes_received += stat.value as u64; - - let address_stats = address_stat_lookup - .entry(server_info.address_id) - .or_insert_with(HashMap::default); - let counter = address_stats - .entry("total_received".to_string()) - .or_insert(0); - *counter += stat.value; - } - None => (), - } - } - - EventName::CheckoutTime { - client_id, - server_id, - } => { - // Update client stats - let app_name = match client_states.get_mut(&client_id) { - Some(client_info) => { - client_info.total_wait_time += stat.value as u64; - client_info.application_name.to_string() - } - None => String::from("Undefined"), - }; - - // Update server stats and address aggergation stats - match server_states.get_mut(&server_id) { - Some(server_info) => { - server_info.application_name = app_name; - - let address_stats = address_stat_lookup - .entry(server_info.address_id) - .or_insert_with(HashMap::default); - let counter = address_stats - .entry("total_wait_time".to_string()) - .or_insert(0); - *counter += stat.value; - - let pool_stats = pool_stat_lookup - .entry(( - server_info.pool_name.clone(), - server_info.username.clone(), - )) - .or_insert_with(HashMap::default); - - // We record max wait in microseconds, we do the pgbouncer second/microsecond split on admin - let old_microseconds = - pool_stats.entry("maxwait_us".to_string()).or_insert(0); - if stat.value > *old_microseconds { - *old_microseconds = stat.value; - } - } - None => (), - } - } - - EventName::ClientRegistered { - client_id, - pool_name, - username, - application_name, - } => { - match client_states.get_mut(&client_id) { - Some(_) => warn!("Client {:?} was double registered!", client_id), - None => { - client_states.insert( - client_id, - ClientInformation { - state: ClientState::Idle, - connect_time: Instant::now(), - client_id, - pool_name: pool_name.clone(), - username: username.clone(), - application_name: application_name.clone(), - total_wait_time: 0, - transaction_count: 0, - query_count: 0, - error_count: 0, - }, - ); - } - }; - } - - EventName::ClientBanError { - client_id, - address_id, - } => { - match client_states.get_mut(&client_id) { - Some(client_info) => { - client_info.state = ClientState::Idle; - client_info.error_count += stat.value as u64; - } - None => warn!("Got event {:?} for unregistered client", stat.name), - } - - // Update address aggregation stats - let address_stats = address_stat_lookup - .entry(address_id) - .or_insert_with(HashMap::default); - let counter = address_stats.entry("total_errors".to_string()).or_insert(0); - *counter += stat.value; - } - - EventName::ClientCheckoutError { - client_id, - address_id, - } => { - match client_states.get_mut(&client_id) { - Some(client_info) => { - client_info.state = ClientState::Idle; - client_info.error_count += stat.value as u64; - } - None => warn!("Got event {:?} for unregistered client", stat.name), - } - - // Update address aggregation stats - let address_stats = address_stat_lookup - .entry(address_id) - .or_insert_with(HashMap::default); - let counter = address_stats.entry("total_errors".to_string()).or_insert(0); - *counter += stat.value; - } - - EventName::ClientIdle { client_id } => { - match client_states.get_mut(&client_id) { - Some(client_state) => client_state.state = ClientState::Idle, - None => warn!("Got event {:?} for unregistered client", stat.name), - }; - } - - EventName::ClientWaiting { client_id } => { - match client_states.get_mut(&client_id) { - Some(client_state) => client_state.state = ClientState::Waiting, - None => warn!("Got event {:?} for unregistered client", stat.name), - }; - } - - EventName::ClientActive { - client_id, - server_id: _, - } => { - match client_states.get_mut(&client_id) { - Some(client_state) => client_state.state = ClientState::Active, - None => warn!("Got event {:?} for unregistered client", stat.name), - }; - } - - EventName::ClientDisconnecting { client_id } => { - client_states.remove(&client_id); - } - - EventName::ServerRegistered { - address_name, - server_id, - address_id, - pool_name, - username, - } => { - server_states.insert( - server_id, - ServerInformation { - address_id, - address_name, - server_id, - username, - pool_name, - - state: ServerState::Idle, - application_name: String::from("Undefined"), - connect_time: Instant::now(), - bytes_sent: 0, - bytes_received: 0, - transaction_count: 0, - query_count: 0, - error_count: 0, - }, - ); - } - - EventName::ServerLogin { server_id } => { - match server_states.get_mut(&server_id) { - Some(server_state) => { - server_state.state = ServerState::Login; - server_state.application_name = String::from("Undefined"); - } - None => warn!("Got event {:?} for unregistered server", stat.name), - }; - } - - EventName::ServerTested { server_id } => { - match server_states.get_mut(&server_id) { - Some(server_state) => { - server_state.state = ServerState::Tested; - server_state.application_name = String::from("Undefined"); - } - None => warn!("Got event {:?} for unregistered server", stat.name), - }; - } - - EventName::ServerIdle { server_id } => { - match server_states.get_mut(&server_id) { - Some(server_state) => { - server_state.state = ServerState::Idle; - server_state.application_name = String::from("Undefined"); - } - None => warn!("Got event {:?} for unregistered server", stat.name), - }; - } - - EventName::ServerActive { - client_id, - server_id, - } => { - // Update client stats - let app_name = match client_states.get_mut(&client_id) { - Some(client_info) => client_info.application_name.to_string(), - None => String::from("Undefined"), - }; - - // Update server stats - match server_states.get_mut(&server_id) { - Some(server_state) => { - server_state.state = ServerState::Active; - server_state.application_name = app_name; - } - None => warn!("Got event {:?} for unregistered server", stat.name), - }; - } - - EventName::ServerDisconnecting { server_id } => { - server_states.remove(&server_id); - } - - EventName::UpdateStats { - pool_name, - username, - } => { - let pool_stats = pool_stat_lookup - .entry((pool_name.clone(), username.clone())) - .or_insert_with(HashMap::default); - - // These are re-calculated every iteration of the loop, so we don't want to add values - // from the last iteration. - for stat in &[ - "cl_active", - "cl_waiting", - "cl_idle", - "sv_idle", - "sv_active", - "sv_tested", - "sv_login", - ] { - pool_stats.insert(stat.to_string(), 0); - } - - for (_, client_info) in client_states.iter() { - if client_info.pool_name != pool_name || client_info.username != username { - continue; - } - match client_info.state { - ClientState::Idle => { - let counter = pool_stats.entry("cl_idle".to_string()).or_insert(0); - *counter += 1; - } - ClientState::Waiting => { - let counter = - pool_stats.entry("cl_waiting".to_string()).or_insert(0); - *counter += 1; - } - ClientState::Active => { - let counter = - pool_stats.entry("cl_active".to_string()).or_insert(0); - *counter += 1; - } - }; - } - - for (_, server_info) in server_states.iter() { - if server_info.pool_name != pool_name || server_info.username != username { - continue; - } - match server_info.state { - ServerState::Login => { - let counter = pool_stats.entry("sv_login".to_string()).or_insert(0); - *counter += 1; - } - ServerState::Tested => { - let counter = - pool_stats.entry("sv_tested".to_string()).or_insert(0); - *counter += 1; - } - ServerState::Active => { - let counter = - pool_stats.entry("sv_active".to_string()).or_insert(0); - *counter += 1; - } - ServerState::Idle => { - let counter = pool_stats.entry("sv_idle".to_string()).or_insert(0); - *counter += 1; - } - }; - } - - // The following calls publish the internal stats making it visible - // to clients using admin database to issue queries like `SHOW STATS` - LATEST_CLIENT_STATS.store(Arc::new(client_states.clone())); - LATEST_SERVER_STATS.store(Arc::new(server_states.clone())); - LATEST_POOL_STATS.store(Arc::new(pool_stat_lookup.clone())); - - // Clear maxwait after reporting - pool_stat_lookup - .entry((pool_name.clone(), username.clone())) - .or_insert_with(HashMap::default) - .insert("maxwait_us".to_string(), 0); - } - - EventName::UpdateAverages { address_id } => { - let stats = address_stat_lookup - .entry(address_id) - .or_insert_with(HashMap::default); - let old_stats = address_old_stat_lookup - .entry(address_id) - .or_insert_with(HashMap::default); - - // Calculate averages - for stat in &[ - "avg_query_count", - "avg_query_time", - "avg_recv", - "avg_sent", - "avg_errors", - "avg_xact_time", - "avg_xact_count", - "avg_wait_time", - ] { - let total_name = match stat { - &"avg_recv" => "total_received".to_string(), // Because PgBouncer is saving bytes - _ => stat.replace("avg_", "total_"), - }; - - let old_value = old_stats.entry(total_name.clone()).or_insert(0); - let new_value = stats.get(total_name.as_str()).unwrap_or(&0).to_owned(); - let avg = (new_value - *old_value) / (STAT_PERIOD as i64 / 1_000); // Avg / second - - stats.insert(stat.to_string(), avg); - *old_value = new_value; - } - LATEST_ADDRESS_STATS.store(Arc::new(address_stat_lookup.clone())); - } - }; - } } } -/// Get a snapshot of client statistics. Updated once a second +/// Get a snapshot of client statistics. /// by the `Collector`. pub fn get_client_stats() -> ClientStatesLookup { - (*(*LATEST_CLIENT_STATS.load())).clone() + CLIENT_STATS.read().clone() } -/// Get a snapshot of server statistics. Updated once a second +/// Get a snapshot of server statistics. /// by the `Collector`. pub fn get_server_stats() -> ServerStatesLookup { - (*(*LATEST_SERVER_STATS.load())).clone() + SERVER_STATS.read().clone() } -/// Get a snapshot of pool statistics. Updated once a second +/// Get a snapshot of pool statistics. /// by the `Collector`. pub fn get_pool_stats() -> PoolStatsLookup { - (*(*LATEST_POOL_STATS.load())).clone() -} - -/// Get a snapshot of address statistics. Updated once a second -/// by the `Collector`. -pub fn get_address_stats() -> AddressStatsLookup { - (*(*LATEST_ADDRESS_STATS.load())).clone() + POOL_STATS.read().clone() } /// Get the statistics reporter used to update stats across the pools/clients. diff --git a/src/stats/address.rs b/src/stats/address.rs new file mode 100644 index 000000000..a5759e100 --- /dev/null +++ b/src/stats/address.rs @@ -0,0 +1,149 @@ +use log::warn; +use std::sync::atomic::*; +use std::sync::Arc; + +/// Internal address stats +#[derive(Debug, Clone, Default)] +pub struct AddressStats { + pub total_xact_count: Arc, + pub total_query_count: Arc, + pub total_received: Arc, + pub total_sent: Arc, + pub total_xact_time: Arc, + pub total_query_time: Arc, + pub total_wait_time: Arc, + pub total_errors: Arc, + pub avg_query_count: Arc, + pub avg_query_time: Arc, + pub avg_recv: Arc, + pub avg_sent: Arc, + pub avg_errors: Arc, + pub avg_xact_time: Arc, + pub avg_xact_count: Arc, + pub avg_wait_time: Arc, +} + +impl IntoIterator for AddressStats { + type Item = (String, u64); + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + vec![ + ( + "total_xact_count".to_string(), + self.total_xact_count.load(Ordering::Relaxed), + ), + ( + "total_query_count".to_string(), + self.total_query_count.load(Ordering::Relaxed), + ), + ( + "total_received".to_string(), + self.total_received.load(Ordering::Relaxed), + ), + ( + "total_sent".to_string(), + self.total_sent.load(Ordering::Relaxed), + ), + ( + "total_xact_time".to_string(), + self.total_xact_time.load(Ordering::Relaxed), + ), + ( + "total_query_time".to_string(), + self.total_query_time.load(Ordering::Relaxed), + ), + ( + "total_wait_time".to_string(), + self.total_wait_time.load(Ordering::Relaxed), + ), + ( + "total_errors".to_string(), + self.total_errors.load(Ordering::Relaxed), + ), + ( + "avg_xact_count".to_string(), + self.avg_xact_count.load(Ordering::Relaxed), + ), + ( + "avg_query_count".to_string(), + self.avg_query_count.load(Ordering::Relaxed), + ), + ( + "avg_recv".to_string(), + self.avg_recv.load(Ordering::Relaxed), + ), + ( + "avg_sent".to_string(), + self.avg_sent.load(Ordering::Relaxed), + ), + ( + "avg_errors".to_string(), + self.avg_errors.load(Ordering::Relaxed), + ), + ( + "avg_xact_time".to_string(), + self.avg_xact_time.load(Ordering::Relaxed), + ), + ( + "avg_query_time".to_string(), + self.avg_query_time.load(Ordering::Relaxed), + ), + ( + "avg_wait_time".to_string(), + self.avg_wait_time.load(Ordering::Relaxed), + ), + ] + .into_iter() + } +} + +impl AddressStats { + pub fn error(&self) { + self.total_errors.fetch_add(1, Ordering::Relaxed); + } + + pub fn update_averages(&self) { + let (totals, averages) = self.fields_iterators(); + for data in totals.iter().zip(averages.iter()) { + let (total, average) = data; + if let Err(err) = average.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |avg| { + let total = total.load(Ordering::Relaxed); + let avg = (total - avg) / (crate::stats::STAT_PERIOD / 1_000); // Avg / second + Some(avg) + }) { + warn!("Could not update averages for addresses stats, {:?}", err); + } + } + } + + pub fn populate_row(&self, row: &mut Vec) { + for (_key, value) in self.clone() { + row.push(value.to_string()); + } + } + + fn fields_iterators(&self) -> (Vec>, Vec>) { + let mut totals: Vec> = Vec::new(); + let mut averages: Vec> = Vec::new(); + + totals.push(self.total_xact_count.clone()); + averages.push(self.avg_xact_count.clone()); + totals.push(self.total_query_count.clone()); + averages.push(self.avg_query_count.clone()); + totals.push(self.total_received.clone()); + averages.push(self.avg_recv.clone()); + totals.push(self.total_sent.clone()); + averages.push(self.avg_sent.clone()); + totals.push(self.total_xact_time.clone()); + averages.push(self.avg_xact_time.clone()); + totals.push(self.total_query_time.clone()); + averages.push(self.avg_query_time.clone()); + totals.push(self.total_wait_time.clone()); + averages.push(self.avg_wait_time.clone()); + totals.push(self.total_errors.clone()); + averages.push(self.avg_errors.clone()); + + (totals, averages) + } +} diff --git a/src/stats/client.rs b/src/stats/client.rs new file mode 100644 index 000000000..892350687 --- /dev/null +++ b/src/stats/client.rs @@ -0,0 +1,182 @@ +use super::PoolStats; +use super::{get_reporter, Reporter}; +use atomic_enum::atomic_enum; +use std::sync::atomic::*; +use std::sync::Arc; +use tokio::time::Instant; +/// The various states that a client can be in +#[atomic_enum] +#[derive(PartialEq)] +pub enum ClientState { + Idle = 0, + Waiting, + Active, +} +impl std::fmt::Display for ClientState { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match *self { + ClientState::Idle => write!(f, "idle"), + ClientState::Waiting => write!(f, "waiting"), + ClientState::Active => write!(f, "active"), + } + } +} + +#[derive(Debug, Clone)] +/// Information we keep track of which can be queried by SHOW CLIENTS +pub struct ClientStats { + /// A random integer assigned to the client and used by stats to track the client + client_id: i32, + + /// Data associated with the client, not writable, only set when we construct the ClientStat + application_name: String, + username: String, + pool_name: String, + connect_time: Instant, + + pool_stats: Arc, + reporter: Reporter, + + /// Total time spent waiting for a connection from pool, measures in microseconds + pub total_wait_time: Arc, + + /// Current state of the client + pub state: Arc, + + /// Number of transactions executed by this client + pub transaction_count: Arc, + + /// Number of queries executed by this client + pub query_count: Arc, + + /// Number of errors made by this client + pub error_count: Arc, +} + +impl Default for ClientStats { + fn default() -> Self { + ClientStats { + client_id: 0, + connect_time: Instant::now(), + application_name: String::new(), + username: String::new(), + pool_name: String::new(), + pool_stats: Arc::new(PoolStats::default()), + total_wait_time: Arc::new(AtomicU64::new(0)), + state: Arc::new(AtomicClientState::new(ClientState::Idle)), + transaction_count: Arc::new(AtomicU64::new(0)), + query_count: Arc::new(AtomicU64::new(0)), + error_count: Arc::new(AtomicU64::new(0)), + reporter: get_reporter(), + } + } +} + +impl ClientStats { + pub fn new( + client_id: i32, + application_name: &str, + username: &str, + pool_name: &str, + connect_time: Instant, + pool_stats: Arc, + ) -> Self { + Self { + client_id, + pool_stats, + connect_time, + application_name: application_name.to_string(), + username: username.to_string(), + pool_name: pool_name.to_string(), + ..Default::default() + } + } + + /// Reports a client is disconecting from the pooler and + /// update metrics on the corresponding pool. + pub fn disconnect(&self) { + self.reporter.client_disconnecting(self.client_id); + self.pool_stats + .client_disconnect(self.state.load(Ordering::Relaxed)) + } + + /// Register a client with the stats system. The stats system uses client_id + /// to track and aggregate statistics from all source that relate to that client + pub fn register(&self, stats: Arc) { + self.reporter.client_register(self.client_id, stats); + self.state.store(ClientState::Idle, Ordering::Relaxed); + self.pool_stats.cl_idle.fetch_add(1, Ordering::Relaxed); + } + + /// Reports a client is done querying the server and is no longer assigned a server connection + pub fn idle(&self) { + self.pool_stats + .client_idle(self.state.load(Ordering::Relaxed)); + self.state.store(ClientState::Idle, Ordering::Relaxed); + } + + /// Reports a client is waiting for a connection + pub fn waiting(&self) { + self.pool_stats + .client_waiting(self.state.load(Ordering::Relaxed)); + self.state.store(ClientState::Waiting, Ordering::Relaxed); + } + + /// Reports a client is done waiting for a connection and is about to query the server. + pub fn active(&self) { + self.pool_stats + .client_active(self.state.load(Ordering::Relaxed)); + self.state.store(ClientState::Active, Ordering::Relaxed); + } + + /// Reports a client has failed to obtain a connection from a connection pool + pub fn checkout_error(&self) { + self.state.store(ClientState::Idle, Ordering::Relaxed); + } + + /// Reports a client has had the server assigned to it be banned + pub fn ban_error(&self) { + self.state.store(ClientState::Idle, Ordering::Relaxed); + self.error_count.fetch_add(1, Ordering::Relaxed); + } + + /// Reportes the time spent by a client waiting to get a healthy connection from the pool + pub fn checkout_time(&self, microseconds: u64) { + self.total_wait_time + .fetch_add(microseconds, Ordering::Relaxed); + } + + /// Report a query executed by a client against a server + pub fn query(&self) { + self.query_count.fetch_add(1, Ordering::Relaxed); + } + + /// Report a transaction executed by a client a server + /// we report each individual queries outside a transaction as a transaction + /// We only count the initial BEGIN as a transaction, all queries within do not + /// count as transactions + pub fn transaction(&self) { + self.transaction_count.fetch_add(1, Ordering::Relaxed); + } + + // Helper methods for show clients + pub fn connect_time(&self) -> Instant { + self.connect_time + } + + pub fn client_id(&self) -> i32 { + self.client_id + } + + pub fn application_name(&self) -> String { + self.application_name.clone() + } + + pub fn username(&self) -> String { + self.username.clone() + } + + pub fn pool_name(&self) -> String { + self.pool_name.clone() + } +} diff --git a/src/stats/pool.rs b/src/stats/pool.rs new file mode 100644 index 000000000..1b01ef2eb --- /dev/null +++ b/src/stats/pool.rs @@ -0,0 +1,274 @@ +use crate::config::Pool; +use crate::config::PoolMode; +use crate::pool::PoolIdentifier; +use std::sync::atomic::*; +use std::sync::Arc; + +use super::get_reporter; +use super::Reporter; +use super::{ClientState, ServerState}; + +#[derive(Debug, Clone, Default)] +/// A struct that holds information about a Pool . +pub struct PoolStats { + // Pool identifier, cannot be changed after creating the instance + identifier: PoolIdentifier, + + // Pool Config, cannot be changed after creating the instance + config: Pool, + + // A reference to the global reporter. + reporter: Reporter, + + /// Counters (atomics) + pub cl_idle: Arc, + pub cl_active: Arc, + pub cl_waiting: Arc, + pub cl_cancel_req: Arc, + pub sv_active: Arc, + pub sv_idle: Arc, + pub sv_used: Arc, + pub sv_tested: Arc, + pub sv_login: Arc, + pub maxwait: Arc, +} + +impl IntoIterator for PoolStats { + type Item = (String, u64); + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + vec![ + ("cl_idle".to_string(), self.cl_idle.load(Ordering::Relaxed)), + ( + "cl_active".to_string(), + self.cl_active.load(Ordering::Relaxed), + ), + ( + "cl_waiting".to_string(), + self.cl_waiting.load(Ordering::Relaxed), + ), + ( + "cl_cancel_req".to_string(), + self.cl_cancel_req.load(Ordering::Relaxed), + ), + ( + "sv_active".to_string(), + self.sv_active.load(Ordering::Relaxed), + ), + ("sv_idle".to_string(), self.sv_idle.load(Ordering::Relaxed)), + ("sv_used".to_string(), self.sv_used.load(Ordering::Relaxed)), + ( + "sv_tested".to_string(), + self.sv_tested.load(Ordering::Relaxed), + ), + ( + "sv_login".to_string(), + self.sv_login.load(Ordering::Relaxed), + ), + ( + "maxwait".to_string(), + self.maxwait.load(Ordering::Relaxed) / 1_000_000, + ), + ( + "maxwait_us".to_string(), + self.maxwait.load(Ordering::Relaxed) % 1_000_000, + ), + ] + .into_iter() + } +} + +impl PoolStats { + pub fn new(identifier: PoolIdentifier, config: Pool) -> Self { + Self { + identifier, + config, + reporter: get_reporter(), + ..Default::default() + } + } + + // Getters + pub fn register(&self, stats: Arc) { + self.reporter.pool_register(self.identifier.clone(), stats); + } + + pub fn database(&self) -> String { + self.identifier.db.clone() + } + + pub fn user(&self) -> String { + self.identifier.user.clone() + } + + pub fn pool_mode(&self) -> PoolMode { + self.config.pool_mode + } + + /// Populates an array of strings with counters (used by admin in show pools) + pub fn populate_row(&self, row: &mut Vec) { + for (_key, value) in self.clone() { + row.push(value.to_string()); + } + } + + /// Deletes the maxwait counter, this is done everytime we obtain metrics + pub fn clear_maxwait(&self) { + self.maxwait.store(0, Ordering::Relaxed); + } + + /// Notified when a server of the pool enters login state. + /// + /// Arguments: + /// + /// `from`: The state of the server that notifies. + pub fn server_login(&self, from: ServerState) { + self.sv_login.fetch_add(1, Ordering::Relaxed); + if from != ServerState::Login { + self.decrease_from_server_state(from); + } + } + + /// Notified when a server of the pool become 'active' + /// + /// Arguments: + /// + /// `from`: The state of the server that notifies. + pub fn server_active(&self, from: ServerState) { + self.sv_active.fetch_add(1, Ordering::Relaxed); + if from != ServerState::Active { + self.decrease_from_server_state(from); + } + } + + /// Notified when a server of the pool become 'tested' + /// + /// Arguments: + /// + /// `from`: The state of the server that notifies. + pub fn server_tested(&self, from: ServerState) { + self.sv_tested.fetch_add(1, Ordering::Relaxed); + if from != ServerState::Tested { + self.decrease_from_server_state(from); + } + } + + /// Notified when a server of the pool become 'idle' + /// + /// Arguments: + /// + /// `from`: The state of the server that notifies. + pub fn server_idle(&self, from: ServerState) { + self.sv_idle.fetch_add(1, Ordering::Relaxed); + if from != ServerState::Idle { + self.decrease_from_server_state(from); + } + } + + /// Notified when a client of the pool become 'waiting' + /// + /// Arguments: + /// + /// `from`: The state of the client that notifies. + pub fn client_waiting(&self, from: ClientState) { + if from != ClientState::Waiting { + self.cl_waiting.fetch_add(1, Ordering::Relaxed); + self.decrease_from_client_state(from); + } + } + + /// Notified when a client of the pool become 'active' + /// + /// Arguments: + /// + /// `from`: The state of the client that notifies. + pub fn client_active(&self, from: ClientState) { + if from != ClientState::Active { + self.cl_active.fetch_add(1, Ordering::Relaxed); + self.decrease_from_client_state(from); + } + } + + /// Notified when a client of the pool become 'idle' + /// + /// Arguments: + /// + /// `from`: The state of the client that notifies. + pub fn client_idle(&self, from: ClientState) { + if from != ClientState::Idle { + self.cl_idle.fetch_add(1, Ordering::Relaxed); + self.decrease_from_client_state(from); + } + } + + /// Notified when a client disconnects. + /// + /// Arguments: + /// + /// `from`: The state of the client that notifies. + pub fn client_disconnect(&self, from: ClientState) { + let counter = match from { + ClientState::Idle => &self.cl_idle, + ClientState::Waiting => &self.cl_waiting, + ClientState::Active => &self.cl_active, + }; + + Self::decrease_counter(counter.clone()); + } + + /// Notified when a server disconnects. + /// + /// Arguments: + /// + /// `from`: The state of the client that notifies. + pub fn server_disconnect(&self, from: ServerState) { + let counter = match from { + ServerState::Active => &self.sv_active, + ServerState::Idle => &self.sv_idle, + ServerState::Login => &self.sv_login, + ServerState::Tested => &self.sv_tested, + }; + Self::decrease_counter(counter.clone()); + } + + // helpers for counter decrease + fn decrease_from_server_state(&self, from: ServerState) { + let counter = match from { + ServerState::Tested => &self.sv_tested, + ServerState::Active => &self.sv_active, + ServerState::Idle => &self.sv_idle, + ServerState::Login => &self.sv_login, + }; + Self::decrease_counter(counter.clone()); + } + + fn decrease_from_client_state(&self, from: ClientState) { + let counter = match from { + ClientState::Active => &self.cl_active, + ClientState::Idle => &self.cl_idle, + ClientState::Waiting => &self.cl_waiting, + }; + Self::decrease_counter(counter.clone()); + } + + fn decrease_counter(value: Arc) { + if value.load(Ordering::Relaxed) > 0 { + value.fetch_sub(1, Ordering::Relaxed); + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_decrease() { + let stat: PoolStats = PoolStats::default(); + stat.server_login(ServerState::Login); + stat.server_idle(ServerState::Login); + assert_eq!(stat.sv_login.load(Ordering::Relaxed), 0); + assert_eq!(stat.sv_idle.load(Ordering::Relaxed), 1); + } +} diff --git a/src/stats/server.rs b/src/stats/server.rs new file mode 100644 index 000000000..009e9b57d --- /dev/null +++ b/src/stats/server.rs @@ -0,0 +1,226 @@ +use super::AddressStats; +use super::PoolStats; +use super::{get_reporter, Reporter}; +use crate::config::Address; +use atomic_enum::atomic_enum; +use parking_lot::RwLock; +use std::sync::atomic::*; +use std::sync::Arc; +use tokio::time::Instant; + +/// The various states that a server can be in +#[atomic_enum] +#[derive(PartialEq)] +pub enum ServerState { + Login = 0, + Active, + Tested, + Idle, +} +impl std::fmt::Display for ServerState { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match *self { + ServerState::Login => write!(f, "login"), + ServerState::Active => write!(f, "active"), + ServerState::Tested => write!(f, "tested"), + ServerState::Idle => write!(f, "idle"), + } + } +} + +/// Information we keep track of which can be queried by SHOW SERVERS +#[derive(Debug, Clone)] +pub struct ServerStats { + /// A random integer assigned to the server and used by stats to track the server + server_id: i32, + + /// Context information, only to be read + address: Address, + connect_time: Instant, + + pool_stats: Arc, + reporter: Reporter, + + /// Data + pub application_name: Arc>, + pub state: Arc, + pub bytes_sent: Arc, + pub bytes_received: Arc, + pub transaction_count: Arc, + pub query_count: Arc, + pub error_count: Arc, +} + +impl Default for ServerStats { + fn default() -> Self { + ServerStats { + server_id: 0, + application_name: Arc::new(RwLock::new(String::new())), + address: Address::default(), + pool_stats: Arc::new(PoolStats::default()), + connect_time: Instant::now(), + state: Arc::new(AtomicServerState::new(ServerState::Login)), + bytes_sent: Arc::new(AtomicU64::new(0)), + bytes_received: Arc::new(AtomicU64::new(0)), + transaction_count: Arc::new(AtomicU64::new(0)), + query_count: Arc::new(AtomicU64::new(0)), + error_count: Arc::new(AtomicU64::new(0)), + reporter: get_reporter(), + } + } +} + +impl ServerStats { + pub fn new(address: Address, pool_stats: Arc, connect_time: Instant) -> Self { + Self { + address, + pool_stats, + connect_time, + server_id: rand::random::(), + ..Default::default() + } + } + + pub fn server_id(&self) -> i32 { + self.server_id + } + + /// Register a server connection with the stats system. The stats system uses server_id + /// to track and aggregate statistics from all source that relate to that server + // Delegates to reporter + pub fn register(&self, stats: Arc) { + self.reporter.server_register(self.server_id, stats); + self.login(); + } + + /// Reports a server connection is no longer assigned to a client + /// and is available for the next client to pick it up + pub fn idle(&self) { + self.pool_stats + .server_idle(self.state.load(Ordering::Relaxed)); + + self.state.store(ServerState::Idle, Ordering::Relaxed); + self.set_undefined_application(); + } + + /// Reports a server connection is disconecting from the pooler. + /// Also updates metrics on the pool regarding server usage. + pub fn disconnect(&self) { + self.reporter.server_disconnecting(self.server_id); + self.pool_stats + .server_disconnect(self.state.load(Ordering::Relaxed)) + } + + /// Reports a server connection is being tested before being given to a client. + pub fn tested(&self) { + self.set_undefined_application(); + self.pool_stats + .server_tested(self.state.load(Ordering::Relaxed)); + self.state.store(ServerState::Tested, Ordering::Relaxed); + } + + /// Reports a server connection is attempting to login. + pub fn login(&self) { + self.pool_stats + .server_login(self.state.load(Ordering::Relaxed)); + self.state.store(ServerState::Login, Ordering::Relaxed); + self.set_undefined_application(); + } + + /// Reports a server connection has been assigned to a client that + /// is about to query the server + pub fn active(&self, application_name: String) { + self.pool_stats + .server_active(self.state.load(Ordering::Relaxed)); + self.state.store(ServerState::Active, Ordering::Relaxed); + self.set_application(application_name); + } + + pub fn address_stats(&self) -> Arc { + self.address.stats.clone() + } + + // Helper methods for show_servers + pub fn pool_name(&self) -> String { + self.pool_stats.database() + } + + pub fn username(&self) -> String { + self.pool_stats.user() + } + + pub fn address_name(&self) -> String { + self.address.name() + } + + pub fn connect_time(&self) -> Instant { + self.connect_time + } + + fn set_application(&self, name: String) { + let mut application_name = self.application_name.write(); + *application_name = name; + } + + fn set_undefined_application(&self) { + self.set_application(String::from("Undefined")) + } + + pub fn checkout_time(&self, microseconds: u64, application_name: String) { + // Update server stats and address aggergation stats + self.set_application(application_name); + self.address + .stats + .total_wait_time + .fetch_add(microseconds, Ordering::Relaxed); + self.pool_stats + .maxwait + .fetch_max(microseconds, Ordering::Relaxed); + } + + /// Report a query executed by a client against a server + pub fn query(&self, milliseconds: u64, application_name: &str) { + self.set_application(application_name.to_string()); + let address_stats = self.address_stats(); + address_stats + .total_query_count + .fetch_add(1, Ordering::Relaxed); + address_stats + .total_query_time + .fetch_add(milliseconds, Ordering::Relaxed); + } + + /// Report a transaction executed by a client a server + /// we report each individual queries outside a transaction as a transaction + /// We only count the initial BEGIN as a transaction, all queries within do not + /// count as transactions + pub fn transaction(&self, application_name: &str) { + self.set_application(application_name.to_string()); + + self.transaction_count.fetch_add(1, Ordering::Relaxed); + self.address + .stats + .total_xact_count + .fetch_add(1, Ordering::Relaxed); + } + + /// Report data sent to a server + pub fn data_sent(&self, amount_bytes: usize) { + self.bytes_sent + .fetch_add(amount_bytes as u64, Ordering::Relaxed); + self.address + .stats + .total_sent + .fetch_add(amount_bytes as u64, Ordering::Relaxed); + } + + /// Report data received from a server + pub fn data_received(&self, amount_bytes: usize) { + self.bytes_received + .fetch_add(amount_bytes as u64, Ordering::Relaxed); + self.address + .stats + .total_received + .fetch_add(amount_bytes as u64, Ordering::Relaxed); + } +} diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index f69c3df6f..b611e8cb3 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -176,6 +176,47 @@ end end + context "clients connects and disconnect normally" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) } + + it 'shows the same number of clients before and after' do + clients_before = clients_connected_to_pool(processes: processes) + threads = [] + connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT 1") } + end + clients_between = clients_connected_to_pool(processes: processes) + expect(clients_before).not_to eq(clients_between) + connections.each(&:close) + clients_after = clients_connected_to_pool(processes: processes) + expect(clients_before).to eq(clients_after) + end + end + + context "clients connects and disconnect abruptly" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 10) } + + it 'shows the same number of clients before and after' do + threads = [] + connections = Array.new(2) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT 1") } + end + clients_before = clients_connected_to_pool(processes: processes) + random_string = (0...8).map { (65 + rand(26)).chr }.join + connection_string = "#{pgcat_conn_str}?application_name=#{random_string}" + faulty_client = Process.spawn("psql -Atx #{connection_string} >/dev/null") + sleep(1) + # psql starts two processes, we only know the pid of the parent, this + # ensure both are killed + `pkill -9 -f '#{random_string}'` + Process.wait(faulty_client) + clients_after = clients_connected_to_pool(processes: processes) + expect(clients_before).to eq(clients_after) + end + end + context "clients overwhelm server pools" do let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) } @@ -199,7 +240,7 @@ sleep(2.5) # Allow time for stats to update results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login].each do |s| raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" end expect(results["cl_idle"]).to eq("4") diff --git a/tests/ruby/capture b/tests/ruby/capture new file mode 100644 index 0000000000000000000000000000000000000000..268680ff5141f12abdc6e19d7056909c5cfd4075 GIT binary patch literal 40638 zcmeHQ33wDm7Vb$%0x zAadw(xCA!B3bKM52rI~aA3}f}C;YSMTWl zUsrW;C~wV@0IiC4<5NYeWzw|OgB$d(*Dgfs1Uqo6$`3FFR?&j#*w&i%gn7F~v(z+w z{i>#!a4X(jyjy#~`Zz#n5wN9RIr-20`2pH<&mU%z&PDv(F(Fzk{D4~-_%Y@$82%*( z{0M8)P&lU!Y|AY02R@Pa=QDi7JwH{4-a8oXSqEG}?@moQo+XS+Y6tO?)`w`{25M|` zuXo@{^G?LKwi59Jx0_A|fUdc4F5=<2_}!Dw1#s1G01ASAuFyX`0g)X0hQwa|GBUO zH~0B($~N!9{MRzX{5Q?DtwekM9nM>3(KNUWc>H-4`iuGRtQURv7|`hiXMa8f?fsq~ z(wk&&Xb#ge!=|DGYl4)RHY?E|#mULcuC0)l!rusNuXWzm#r+$^UA7!0qj}M9yZ4JcF z?j_Z<)^GxD_Whxn$rK#$^p!Oq)MCF!Cmg`|8M7t(cFmN1yUlid?wAGIT|K&Wi0ab1 zW#?A05r2+|?b@}|!_keR!$T3Y-TOK~4>Hir1iEPSn1yA6t^?2(57y9TgO<46&@@2Q zbliq17R^-i-h&8w*=GASH!EbQmb_rPTTNr1>(bR6(WGhfMopSEvu};f(ajq-4iD|n zvZFa7G9#mZnz?^cMndZF#PrDUP&_^>EhWvIo|>B3KXG_cLLwa6!yM5pJTyFXJt%p6 zsk-Nh&tlJC>7c9U2-;+s4$xQz+MYnAKH3PN#Vd4xf;mCcPy&_uNH^$I23n0krJ8O6 zP4{05&|Uh zQ&JKWGOP5?Oh^kFnwrpesHTN!CTz@fJwO`<=ioLmAJ5hxA8tfhpWwnnTCsYo= z)1JO!vqGlW?AKBIuXY9A+QYuvZwR^+xTh=yj%AK$)ko9%+>+hXL4jkLHY@CLufV!Y zj}>_Lo?vz5vICyD;-1^!f4PK>15u8ICMDkoRlQ^iEm-z%qFh@XTgbv=?7i1o;y;*pm6 zydK#MN@lHAv;0AKB54`*iY`KqplkGi)?uKd2~=7-Yyr^gKZR(kAhK{PgXPcLdOx!K zdUumKi`qhY_xggRUMlafl7kACRe9+tQ26y78kL+iG!qKI^u)xJsMPd<=8VL2Ec@6_ zGaMWqipux-G)(&&D303=V(oV?cRhCYw??%;hqiuRqjqUh0k-whT53ybeFP0& zD+LXP#{=9Yb12 z5#Rc#h%Xjqh-VaLnnpKplE#o0kLU9KrJ(?l#*kKdEy>R#JCelwe-rXwS(qKe{MQDG z`7ah`nEzB9c%q?En0aC?*JH`%B+;LX!Jo>)>^S<1JAG4Qxp_kkH>F6a4 zMnEq+{xN&G4|*B>A00JFcf8`g^dyxF(&|eM8kG!b5S`i{K{H|rR9b@|=m+11DF=s& z1|Vq9Sh_Y(7%*XgSc4?@FJZMuhr9xZ8U*oYH4Ik{4iyb#_|G}uyQx8X&ExGL?&+rC zYP|KK4$Ae* zBlfdMR>ritjJR^CuInT+7s;DpaA+3>q1`&pQkT&r2hpGqNgNz<*PaBzqg~GW`+8a( z=Knj8|Ee0KXfWn~_d{a-d#eV){CBnkefP+iZVo0tlB|qrb(sIuAF35dx>4Dqp9qH} z`g0lhQ&oc$4PpLF5dG<`8bs`Gh)2Cq9~^8ygk)t*tMg^N`g&R&#?Oa|YTmm49P-v% zV*Z%CbwBiS@;xRctIC;=Su(8_Wy$< zu|XSv5}WZyHn`ScgX+tC8As4vlq5cpo`^wcKi*lE0i*k8Vhtpb4Z3SjH=yD`XZ?N6 z1~LCPK>n-PV9r9!|0#>b{Pz|c#Qb-*1ASMRPq~}?NU}26Am%^yhgcg)Wd{@o5DrQ7 z=U(uqiVfy0V*Y$s^ryGjpx9r3WP`qpS6{P1jGyG~Rg^p_J{RMsp!8+Yl~-3@6?xLE zG2b%&GA#d`3s1WCQxzpoir>ueOGiu?z8juYYc+2N_1?*LE_$A4k;i(xbV(d1f`wz(qn3A1J z_fwL{liame(RS#3(wk&u@T3gLe-%%P--`KvW|Wx!-r`BVdB*j5_ zCE<|7`^9+hr-~=V|Bv~zn&?k&@g%Xo{>YPj8Lz(PNrxbQW<9E&FnYL$1)iE!4Wg%J zKAkXn7?hl_Q9W3j&L)hel};Fyj;pFc`GnEb4wyL^&(dJ6bOiWI&@^(3TGOmx1Wlen zpi)hBfU=pwSwReRCV@&dJpr1ItzK0^kkb=ktojM_mk4S)OcxI4)mnmHn6?V1ww~QE zfl0csA^xL}SCtUtj5NTHF`r=gCmrwwg1p?q5ep{qb`ZCYMW(lAVtSusxCb3@g$Idy zrg6N_61k*y5I-xas*)g^^AX<~E#iv=8S%U^we?G4dSuuNN*YckSsBC1#%e9)cOnrB zlE&25^4k*6Bs-F1*G0&G6+t$i!u;3ziuo@RWXyjeX`X0kOl|eVTCT^E%}Js^CxSm! z1lfEV{b{=_`qNtk*;uW`)lYhVp~9S`uL<%gh@U}EstNL`JS520d6hztPlJ*Jx2kF9 zaJCS`kgd8Hq8gOXW*p8|Ul@`{pi(Y-1~kpC8?Jn9Zv=a#d+QO~93M9}-Cpu7qjJ&b|gOQ2G7o(IsV*JQ81MTfH*;S7SB_KMeh@!|h# zIccl0w&_-|fmAjzBI;lHsG zrQYapJ&tM`l9>Myb#>G>#hCxv3u69@wGHM!@oU#s%l^fvZOXQdo;_r9lIYJA@Tao2 zIg9?Zm5BZ{Y8zLJ2iboj>F=-YU4rqKbHEp<{89@?ytt6JgSdwm$n>^Td6U6^1E($>w+^uX9Q5Ab$02I+jK(L40dN5nrV8h-XlFV+9V%2X9HDoFt85ii%m5 z>_`&xzdq!@a+u;g=D!vv=D$egG5?89dZM90M>#<~WlIYI{;7=u$zkvR<6^Z^d zsJyFviNcfgcPf7!;%8~Fnjl|=*(|1-=V~DN@;O09B^MURXeWCv8$F3FOBVkbF+?es zRf85}BkP|0B4$qfqXa6YomSAa_fhpQ#T)$)v}c7@7YrEEp#PadnN>I$pJKtbdUVy* z0W~Nm<5OBN&^Q8>nmPnDy##DgRfS#Oh(UIGF@pL$$WY1juVm)rK{oAU?0MX2>bW!? z)SyYLG03L0WajKbpi*;I1x-I1rsHw{MFchFiPwCwwwaq?&cWK|r7caEq^lv~Pkm3v zFvT|ve}@CUP}|VE0~s@UJBZtJi4MIlG2G1#xI%67{97EaZ5WJi*i|8GM6D{Gr?G5@vF6^U>dRIF_<|A}9F zqM=dScw#NrW69XMfM_|xj56ozYYz!eysrP>&# zZ{tFaON0>4#w+2FM1O7of2tT= z5u3Hy{YBBA-ePoOe?uzTL8tSB?cYfnYqdEU$iuJr6b3&e!|{3$5+LClp=^R~&SK;P z)2dVaIIm*%$zR7YzA^WMkq$fw@!x+UK#PSoakIDU{9n<>ZRXiyNE*P+i*?ckoy~UF-|H*&O z(yfy`_>ufbvNCv*xT+WN8Im4!#qFVlLlXVD4*aR&NjWeL1N`}*=udC)B(cB#$dh~- zufBfy5aXxYnji^J;(R0ibBv#Y(*L;Gc{Q+CFHd0nsCP&HT!w-d@z z@6VHrRm><$JeSV*O&lR~x2hN&=Xnc`XVCqWq`$9XhS!&~kpC*46u$$nFK05v{Pz}5 z^7rRS;;LT6XGl6mpXc8Qha~#*0{BzKlj3(We+G*F^cGJN`|FQ9$(Ql!Yo1gS;%8}? zdSY7ORGgT0`~VK-`Fvs;D*4G}^)Tivwgi5spJ~KVI*h3X<-?e>*n0h)ej!llFy04LSn!eoz)3IYa0+nitpey8{?7gtJ+cMC$1S-|^P5|xHMz$Ile~@Utf}o~Nbm4G* zWByS6h>vgz-M?S1!z5kO5dWPqvQ^9&0go|%&+xZ8;0q5Dms>buWM|$E;-*iR>CK6~ z`3HvksROR?AhC8|ju&L+lG;K1(HnHU>3pq2(lZ=o8dvC@ut5R0iOMv3JH>!{~tpBs|d3BN6de%bfSp!6gtd*B59sz zWK5y+#9FS$lFdn?KR1CtRRr1m6Z+G3RJ@*hiy#}Tn7R5%?=N)!BY+{H~+KHgw$w3+IoWov}ZdbbCn3Q(b2hgtz z)vpK5eFoF9ZRtBlLEVjN-<=ugfY3`i&fvS!EvMwXsVGRBDxm0NSRu RI>>4?#30-ASfvEn{{c}CVT}L) literal 0 HcmV?d00001 diff --git a/tests/ruby/spec_helper.rb b/tests/ruby/spec_helper.rb index d67964017..a95969fd9 100644 --- a/tests/ruby/spec_helper.rb +++ b/tests/ruby/spec_helper.rb @@ -19,3 +19,10 @@ def with_captured_stdout_stderr STDOUT.reopen(sout) STDERR.reopen(serr) end + +def clients_connected_to_pool(pool_index: 0, processes:) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[pool_index] + admin_conn.close + results['cl_idle'].to_i + results['cl_active'].to_i + results['cl_waiting'].to_i +end From 0f7758d9944d6ca3318dd48f549b63966b30d06b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Mar 2023 09:54:31 -0700 Subject: [PATCH 368/551] chore(deps): bump serde_derive from 1.0.154 to 1.0.159 (#387) Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.154 to 1.0.159. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.154...v1.0.159) --- updated-dependencies: - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e4aa6838..5374d4139 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -978,13 +978,13 @@ checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" [[package]] name = "serde_derive" -version = "1.0.154" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc80d722935453bcafdc2c9a73cd6fac4dc1938f0346035d84bf99fa9e33217" +checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.9", ] [[package]] From 1ef7b2806573d9dd7ec3c759d5341b08d3644ddd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Mar 2023 09:54:39 -0700 Subject: [PATCH 369/551] chore(deps): bump serde from 1.0.158 to 1.0.159 (#386) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.158 to 1.0.159. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.158...v1.0.159) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5374d4139..a4e53465a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -972,9 +972,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.158" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" +checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" [[package]] name = "serde_derive" From e9c39b952079aa31ea61eb081a0e6551c1d579dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Thu, 30 Mar 2023 22:29:23 +0200 Subject: [PATCH 370/551] Auth passthrough (auth_query) (#266) * Add a new exec_simple_query method This adds a new `exec_simple_query` method so we can make 'out of band' queries to servers that don't interfere with pools at all. In order to reuse startup code for making these simple queries, we need to set the stats (`Reporter`) optional, so using these simple queries wont interfere with stats. * Add auth passthough (auth_query) Adds a feature that allows setting auth passthrough for md5 auth. It adds 3 new (general and pool) config parameters: - `auth_query`: An string containing a query that will be executed on boot to obtain the hash of a given user. This query have to use a placeholder `$1`, so pgcat can replace it with the user its trying to fetch the hash from. - `auth_query_user`: The user to use for connecting to the server and executing the auth_query. - `auth_query_password`: The password to use for connecting to the server and executing the auth_query. The configuration can be done either on the general config (so pools share them) or in a per-pool basis. The behavior is, at boot time, when validating server connections, a hash is fetched per server and stored in the pool. When new server connections are created, and no cleartext password is specified, the obtained hash is used for creating them, if the hash could not be obtained for whatever reason, it retries it. When client authentication is tried, it uses cleartext passwords if specified, it not, it checks whether we have query_auth set up, if so, it tries to use the obtained hash for making client auth. If there is no hash (we could not obtain one when validating the connection), a new fetch is tried. Once we have a hash, we authenticate using it against whathever the client has sent us, if there is a failure we refetch the hash and retry auth (so password changes can be done). The idea with this 'retrial' mechanism is to make it fault tolerant, so if for whatever reason hash could not be obtained during connection validation, or the password has change, we can still connect later. * Add documentation for Auth passthrough --- .circleci/config.yml | 8 + .circleci/run_tests.sh | 1 + .rustfmt.toml | 2 + CONFIG.md | 56 +++++- Cargo.lock | 44 ++++- Cargo.toml | 3 +- README.md | 1 + dev/docker-compose.yaml | 8 + src/auth_passthrough.rs | 107 ++++++++++++ src/client.rs | 82 ++++++++- src/config.rs | 89 +++++++++- src/errors.rs | 2 + src/lib.rs | 1 + src/main.rs | 1 + src/messages.rs | 22 ++- src/mirrors.rs | 2 + src/pool.rs | 53 +++++- src/query_router.rs | 6 + src/server.rs | 150 ++++++++++++++++- tests/docker/docker-compose.yml | 9 + tests/ruby/auth_query_spec.rb | 215 ++++++++++++++++++++++++ tests/ruby/helpers/auth_query_helper.rb | 173 +++++++++++++++++++ tests/ruby/helpers/pgcat_helper.rb | 7 + tests/ruby/helpers/pgcat_process.rb | 15 +- 24 files changed, 1026 insertions(+), 31 deletions(-) create mode 100644 .rustfmt.toml create mode 100644 src/auth_passthrough.rs create mode 100644 tests/ruby/auth_query_spec.rb create mode 100644 tests/ruby/helpers/auth_query_helper.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e2d114a2..c7f5c9fa8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,6 +46,14 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 + - image: postgres:14 + command: ["postgres", "-p", "10432", "-c", "shared_preload_libraries=pg_stat_statements"] + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 + # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index a5cfab0b1..4ba497c31 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -19,6 +19,7 @@ PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 5432 -U postgres -f tests/sharding/q PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 7432 -U postgres -f tests/sharding/query_routing_setup.sql PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 8432 -U postgres -f tests/sharding/query_routing_setup.sql PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 9432 -U postgres -f tests/sharding/query_routing_setup.sql +PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 10432 -U postgres -f tests/sharding/query_routing_setup.sql PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard0 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 000000000..17f332112 --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,2 @@ +edition = "2021" +hard_tabs = false diff --git a/CONFIG.md b/CONFIG.md index bcd6f09ff..3cec25301 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -175,11 +175,41 @@ Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DAT ### admin_password ``` path: general.admin_password -default: "admin_pass" +default: ``` Password to access the virtual administrative database +### auth_query (experimental) +``` +path: general.auth_query +default: +``` + +Query to be sent to servers to obtain the hash used for md5 authentication. The connection will be +established using the database configured in the pool. This parameter is inherited by every pool +and can be redefined in pool configuration. + +### auth_query_user (experimental) +``` +path: general.auth_query_user +default: +``` + +User to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +specified in `auth_query_user`. The connection will be established using the database configured in the pool. +This parameter is inherited by every pool and can be redefined in pool configuration. + +### auth_query_password (experimental) +``` +path: general.auth_query_password +default: +``` + +Password to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +specified in `auth_query_user`. The connection will be established using the database configured in the pool. +This parameter is inherited by every pool and can be redefined in pool configuration. + ## `pools.` Section ### pool_mode @@ -281,6 +311,30 @@ default: 3000 Connect timeout can be overwritten in the pool +### auth_query (experimental) +``` +path: general.auth_query +default: +``` + +Auth query can be overwritten in the pool + +### auth_query_user (experimental) +``` +path: general.auth_query_user +default: +``` + +Auth query user can be overwritten in the pool + +### auth_query_password (experimental) +``` +path: general.auth_query_password +default: +``` + +Auth query password can be overwritten in the pool + ## `pools..users.` Section ### username diff --git a/Cargo.lock b/Cargo.lock index a4e53465a..ad7bb8ea9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,7 +45,7 @@ checksum = "6227a8d6fdb862bcb100c4314d0d9579e5cd73fa6df31a2e6f6e1acd3c5f1207" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -54,6 +54,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.0" @@ -94,6 +100,12 @@ version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.4.0" @@ -257,6 +269,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fnv" version = "1.0.7" @@ -732,12 +750,13 @@ dependencies = [ "arc-swap", "async-trait", "atomic_enum", - "base64", + "base64 0.21.0", "bb8", "bytes", "chrono", "env_logger", "exitcode", + "fallible-iterator", "futures", "hmac", "hyper", @@ -749,6 +768,7 @@ dependencies = [ "once_cell", "parking_lot", "phf", + "postgres-protocol", "rand", "regex", "rustls-pemfile", @@ -818,6 +838,24 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "postgres-protocol" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878c6cbf956e03af9aa8204b407b9cbf47c072164800aa918c516cd4b056c50c" +dependencies = [ + "base64 0.13.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand", + "sha2", + "stringprep", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -945,7 +983,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64", + "base64 0.21.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3f0cbec64..89cfe6430 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,6 @@ version = "1.0.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] tokio = { version = "1", features = ["full"] } bytes = "1" @@ -38,6 +37,8 @@ futures = "0.3" socket2 = { version = "0.4.7", features = ["all"] } nix = "0.26.2" atomic_enum = "0.2.0" +postgres-protocol = "0.6.4" +fallible-iterator = "0.2" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/README.md b/README.md index 4d6f599d2..63b5ab1a6 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load bal | Sharding using comments parsing/Regex | **Experimental** | Clients can include shard information (sharding key, shard ID) in the query comments. | | Automatic sharding | **Experimental** | PgCat can parse queries, detect sharding keys automatically, and route queries to the correct shard. | | Mirroring | **Experimental** | Mirror queries between multiple databases in order to test servers with realistic production traffic. | +| Auth passthrough | **Experimental** | MD5 password authentication can be configured to use an `auth_query` so no cleartext passwords are needed in the config file. | ## Status diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 15621e87b..71704bcbc 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -58,6 +58,13 @@ services: POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 PGPORT: 9432 command: ["postgres", "-p", "9432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] + pg5: + <<: *common-definition-pg + environment: + <<: *common-env-pg + POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 + PGPORT: 10432 + command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] toxiproxy: build: . @@ -71,6 +78,7 @@ services: - pg2 - pg3 - pg4 + - pg5 pgcat-shell: stdin_open: true diff --git a/src/auth_passthrough.rs b/src/auth_passthrough.rs new file mode 100644 index 000000000..b9f0e97fb --- /dev/null +++ b/src/auth_passthrough.rs @@ -0,0 +1,107 @@ +use crate::errors::Error; +use crate::server::Server; +use log::debug; + +#[derive(Clone, Debug)] +pub struct AuthPassthrough { + password: String, + query: String, + user: String, +} + +impl AuthPassthrough { + /// Initializes an AuthPassthrough. + pub fn new(query: &str, user: &str, password: &str) -> Self { + AuthPassthrough { + password: password.to_string(), + query: query.to_string(), + user: user.to_string(), + } + } + + /// Returns an AuthPassthrough given the pool configuration. + /// If any of required values is not set, None is returned. + pub fn from_pool_config(pool_config: &crate::config::Pool) -> Option { + if pool_config.is_auth_query_configured() { + return Some(AuthPassthrough::new( + pool_config.auth_query.as_ref().unwrap(), + pool_config.auth_query_user.as_ref().unwrap(), + pool_config.auth_query_password.as_ref().unwrap(), + )); + } + + None + } + + /// Returns an AuthPassthrough given the pool settings. + /// If any of required values is not set, None is returned. + pub fn from_pool_settings(pool_settings: &crate::pool::PoolSettings) -> Option { + let pool_config = crate::config::Pool { + auth_query: pool_settings.auth_query.clone(), + auth_query_password: pool_settings.auth_query_password.clone(), + auth_query_user: pool_settings.auth_query_user.clone(), + ..Default::default() + }; + + AuthPassthrough::from_pool_config(&pool_config) + } + + /// Connects to server and executes auth_query for the specified address. + /// If the response is a row with two columns containing the username set in the address. + /// and its MD5 hash, the MD5 hash returned. + /// + /// Note that the query is executed, changing $1 with the name of the user + /// this is so we only hold in memory (and transfer) the least amount of 'sensitive' data. + /// Also, it is compatible with pgbouncer. + /// + /// # Arguments + /// + /// * `address` - An Address of the server we want to connect to. The username for the hash will be obtained from this value. + /// + /// # Examples + /// + /// ``` + /// use pgcat::auth_passthrough::AuthPassthrough; + /// use pgcat::config::Address; + /// let auth_passthrough = AuthPassthrough::new("SELECT * FROM public.user_lookup('$1');", "postgres", "postgres"); + /// auth_passthrough.fetch_hash(&Address::default()); + /// ``` + /// + pub async fn fetch_hash(&self, address: &crate::config::Address) -> Result { + let auth_user = crate::config::User { + username: self.user.clone(), + password: Some(self.password.clone()), + pool_size: 1, + statement_timeout: 0, + }; + + let user = &address.username; + + debug!("Connecting to server to obtain auth hashes."); + let auth_query = self.query.replace("$1", user); + match Server::exec_simple_query(address, &auth_user, &auth_query).await { + Ok(password_data) => { + if password_data.len() == 2 && password_data.first().unwrap() == user { + if let Some(stripped_hash) = password_data.last().unwrap().to_string().strip_prefix("md5") { + Ok(stripped_hash.to_string()) + } + else { + Err(Error::AuthPassthroughError( + "Obtained hash from auth_query does not seem to be in md5 format.".to_string(), + )) + } + } else { + Err(Error::AuthPassthroughError( + "Data obtained from query does not follow the scheme 'user','hash'." + .to_string(), + )) + } + } + Err(err) => { + Err(Error::AuthPassthroughError( + format!("Error trying to obtain password from auth_query, ignoring hash for user '{}'. Error: {:?}", + user, err))) + } + } + } +} diff --git a/src/client.rs b/src/client.rs index f9f4e015d..d75c069dd 100644 --- a/src/client.rs +++ b/src/client.rs @@ -12,9 +12,9 @@ use tokio::sync::broadcast::Receiver; use tokio::sync::mpsc::Sender; use crate::admin::{generate_server_info_for_admin, handle_admin}; +use crate::auth_passthrough::AuthPassthrough; use crate::config::{get_config, get_idle_client_in_transaction_timeout, Address, PoolMode}; use crate::constants::*; - use crate::messages::*; use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; use crate::query_router::{Command, QueryRouter}; @@ -377,6 +377,20 @@ pub async fn startup_tls( } } +async fn refetch_auth_hash(pool: &ConnectionPool) -> Result { + let address = pool.address(0, 0); + if let Some(apt) = AuthPassthrough::from_pool_settings(&pool.settings) { + let hash = apt.fetch_hash(address).await?; + + return Ok(hash); + } + + Err(Error::ClientError(format!( + "Could not obtain hash for {{ username: {:?}, database: {:?} }}. Auth passthrough not enabled.", + address.username, address.database + ))) +} + impl Client where S: tokio::io::AsyncRead + std::marker::Unpin, @@ -509,14 +523,68 @@ where } }; - // Compare server and client hashes. - let password_hash = md5_hash_password(username, &pool.settings.user.password, &salt); + // Obtain the hash to compare, we give preference to that written in cleartext in config + // if there is nothing set in cleartext and auth passthrough (auth_query) is configured, we use the hash obtained + // when the pool was created. If there is no hash there, we try to fetch it one more time. + let password_hash = if let Some(password) = &pool.settings.user.password { + Some(md5_hash_password(username, password, &salt)) + } else { + if !get_config().is_auth_query_configured() { + return Err(Error::ClientError(format!("Client auth not possible, no cleartext password set for username: {:?} in config and auth passthrough (query_auth) is not set up.", username))); + } - if password_hash != password_response { - warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name); - wrong_password(&mut write, username).await?; + let mut hash = (*pool.auth_hash.read()).clone(); - return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))); + if hash.is_none() { + warn!("Query auth configured but no hash password found for pool {}. Will try to refetch it.", pool_name); + match refetch_auth_hash(&pool).await { + Ok(fetched_hash) => { + warn!("Password for {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}, obtained. Updating.", username, pool_name, application_name); + { + let mut pool_auth_hash = pool.auth_hash.write(); + *pool_auth_hash = Some(fetched_hash.clone()); + } + + hash = Some(fetched_hash); + } + Err(err) => { + return Err( + Error::ClientError( + format!("No cleartext password set, and no auth passthrough could not obtain the hash from server for {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}, the error was: {:?}", + username, + pool_name, + application_name, + err) + ) + ); + } + } + }; + + Some(md5_hash_second_pass(&hash.unwrap(), &salt)) + }; + + // Once we have the resulting hash, we compare with what the client gave us. + // If they do not match and auth query is set up, we try to refetch the hash one more time + // to see if the password has changed since the pool was created. + // + // @TODO: we could end up fetching again the same password twice (see above). + if password_hash.unwrap() != password_response { + warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}, will try to refetch it.", username, pool_name, application_name); + let fetched_hash = refetch_auth_hash(&pool).await?; + let new_password_hash = md5_hash_second_pass(&fetched_hash, &salt); + + // Ok password changed in server an auth is possible. + if new_password_hash == password_response { + warn!("Password for {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}, changed in server. Updating.", username, pool_name, application_name); + { + let mut pool_auth_hash = pool.auth_hash.write(); + *pool_auth_hash = Some(fetched_hash); + } + } else { + wrong_password(&mut write, username).await?; + return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))); + } } let transaction_mode = pool.settings.pool_mode == PoolMode::Transaction; diff --git a/src/config.rs b/src/config.rs index 644532ac1..6545457c6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -177,7 +177,7 @@ impl Address { #[derive(Clone, PartialEq, Hash, Eq, Serialize, Deserialize, Debug)] pub struct User { pub username: String, - pub password: String, + pub password: Option, pub pool_size: u32, #[serde(default)] // 0 pub statement_timeout: u64, @@ -187,7 +187,7 @@ impl Default for User { fn default() -> User { User { username: String::from("postgres"), - password: String::new(), + password: None, pool_size: 15, statement_timeout: 0, } @@ -250,6 +250,10 @@ pub struct General { pub tls_private_key: Option, pub admin_username: String, pub admin_password: String, + + pub auth_query: Option, + pub auth_query_user: Option, + pub auth_query_password: Option, } impl General { @@ -334,6 +338,9 @@ impl Default for General { tls_private_key: None, admin_username: String::from("admin"), admin_password: String::from("admin"), + auth_query: None, + auth_query_user: None, + auth_query_password: None, } } } @@ -406,6 +413,10 @@ pub struct Pool { pub shard_id_regex: Option, pub regex_search_limit: Option, + pub auth_query: Option, + pub auth_query_user: Option, + pub auth_query_password: Option, + pub shards: BTreeMap, pub users: BTreeMap, // Note, don't put simple fields below these configs. There's a compatability issue with TOML that makes it @@ -420,6 +431,12 @@ impl Pool { s.finish() } + pub fn is_auth_query_configured(&self) -> bool { + self.auth_query_password.is_some() + && self.auth_query_user.is_some() + && self.auth_query_password.is_some() + } + pub fn default_pool_mode() -> PoolMode { PoolMode::Transaction } @@ -512,6 +529,9 @@ impl Default for Pool { sharding_key_regex: None, shard_id_regex: None, regex_search_limit: Some(1000), + auth_query: None, + auth_query_user: None, + auth_query_password: None, } } } @@ -612,9 +632,31 @@ pub struct Config { } impl Config { + pub fn is_auth_query_configured(&self) -> bool { + self.pools + .iter() + .any(|(_name, pool)| pool.is_auth_query_configured()) + } + pub fn default_path() -> String { String::from("pgcat.toml") } + + pub fn fill_up_auth_query_config(&mut self) { + for (_name, pool) in self.pools.iter_mut() { + if pool.auth_query.is_none() { + pool.auth_query = self.general.auth_query.clone(); + } + + if pool.auth_query_user.is_none() { + pool.auth_query_user = self.general.auth_query_user.clone(); + } + + if pool.auth_query_password.is_none() { + pool.auth_query_password = self.general.auth_query_password.clone(); + } + } + } } impl Default for Config { @@ -832,6 +874,35 @@ impl Config { } pub fn validate(&mut self) -> Result<(), Error> { + // Validation for auth_query feature + if self.general.auth_query.is_some() + && (self.general.auth_query_user.is_none() + || self.general.auth_query_password.is_none()) + { + error!("If auth_query is specified, you need to provide a value for `auth_query_user`, `auth_query_password`"); + return Err(Error::BadConfig); + } + + for (name, pool) in self.pools.iter() { + if pool.auth_query.is_some() + && (pool.auth_query_user.is_none() || pool.auth_query_password.is_none()) + { + error!("Error in pool {{ {} }}. If auth_query is specified, you need to provide a value for `auth_query_user`, `auth_query_password`", name); + return Err(Error::BadConfig); + } + + for (_name, user_data) in pool.users.iter() { + if (pool.auth_query.is_none() + || pool.auth_query_password.is_none() + || pool.auth_query_user.is_none()) + && user_data.password.is_none() + { + error!("Error in pool {{ {} }}. You have to specify a user password for every pool if auth_query is not specified", name); + return Err(Error::BadConfig); + } + } + } + // Validate TLS! match self.general.tls_certificate.clone() { Some(tls_certificate) => { @@ -911,6 +982,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { } }; + config.fill_up_auth_query_config(); config.validate()?; config.path = path.to_string(); @@ -980,7 +1052,10 @@ mod test { "sharding_user" ); assert_eq!( - get_config().pools["sharded_db"].users["1"].password, + get_config().pools["sharded_db"].users["1"] + .password + .as_ref() + .unwrap(), "other_user" ); assert_eq!(get_config().pools["sharded_db"].users["1"].pool_size, 21); @@ -1005,10 +1080,16 @@ mod test { "simple_user" ); assert_eq!( - get_config().pools["simple_db"].users["0"].password, + get_config().pools["simple_db"].users["0"] + .password + .as_ref() + .unwrap(), "simple_user" ); assert_eq!(get_config().pools["simple_db"].users["0"].pool_size, 5); + assert_eq!(get_config().general.auth_query, None); + assert_eq!(get_config().general.auth_query_user, None); + assert_eq!(get_config().general.auth_query_password, None); } #[tokio::test] diff --git a/src/errors.rs b/src/errors.rs index 310243c09..58fc088b5 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -15,4 +15,6 @@ pub enum Error { StatementTimeout, ShuttingDown, ParseBytesError(String), + AuthError(String), + AuthPassthroughError(String), } diff --git a/src/lib.rs b/src/lib.rs index 67aa9cba0..2645cd427 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod auth_passthrough; pub mod config; pub mod constants; pub mod errors; diff --git a/src/main.rs b/src/main.rs index a59da2108..4c8987f14 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,6 +61,7 @@ use std::sync::Arc; use tokio::sync::broadcast; mod admin; +mod auth_passthrough; mod client; mod config; mod constants; diff --git a/src/messages.rs b/src/messages.rs index c9ace4e0b..61c36c6d7 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -213,7 +213,13 @@ pub fn md5_hash_password(user: &str, password: &str, salt: &[u8]) -> Vec { let output = md5.finalize_reset(); // Second pass - md5.update(format!("{:x}", output)); + md5_hash_second_pass(&(format!("{:x}", output)), salt) +} + +pub fn md5_hash_second_pass(hash: &str, salt: &[u8]) -> Vec { + let mut md5 = Md5::new(); + // Second pass + md5.update(hash); md5.update(salt); let mut password = format!("md5{:x}", md5.finalize()) @@ -247,6 +253,20 @@ where write_all(stream, message).await } +pub async fn md5_password_with_hash(stream: &mut S, hash: &str, salt: &[u8]) -> Result<(), Error> +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let password = md5_hash_second_pass(hash, salt); + let mut message = BytesMut::with_capacity(password.len() as usize + 5); + + message.put_u8(b'p'); + message.put_i32(password.len() as i32 + 4); + message.put_slice(&password[..]); + + write_all(stream, message).await +} + /// Implements a response to our custom `SET SHARDING KEY` /// and `SET SERVER ROLE` commands. /// This tells the client we're ready for the next query. diff --git a/src/mirrors.rs b/src/mirrors.rs index 128fe220c..17f91d4d1 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -4,6 +4,7 @@ use std::sync::Arc; /// Packets arrive to us through a channel from the main client and we send them to the server. use bb8::Pool; use bytes::{Bytes, BytesMut}; +use parking_lot::RwLock; use crate::config::{get_config, Address, Role, User}; use crate::pool::{ClientServerMap, PoolIdentifier, ServerPool}; @@ -41,6 +42,7 @@ impl MirroredClient { self.database.as_str(), ClientServerMap::default(), Arc::new(PoolStats::new(identifier, cfg.clone())), + Arc::new(RwLock::new(None)), ); Pool::builder() diff --git a/src/pool.rs b/src/pool.rs index f6f9118b4..e1ab7cb4e 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -20,6 +20,7 @@ use tokio::sync::Notify; use crate::config::{get_config, Address, General, LoadBalancingMode, PoolMode, Role, User}; use crate::errors::Error; +use crate::auth_passthrough::AuthPassthrough; use crate::server::Server; use crate::sharding::ShardingFunction; use crate::stats::{AddressStats, ClientStats, PoolStats, ServerStats}; @@ -123,6 +124,11 @@ pub struct PoolSettings { // Limit how much of each query is searched for a potential shard regex match pub regex_search_limit: usize, + + // Auth query parameters + pub auth_query: Option, + pub auth_query_user: Option, + pub auth_query_password: Option, } impl Default for PoolSettings { @@ -143,6 +149,9 @@ impl Default for PoolSettings { sharding_key_regex: None, shard_id_regex: None, regex_search_limit: 1000, + auth_query: None, + auth_query_user: None, + auth_query_password: None, } } } @@ -183,6 +192,9 @@ pub struct ConnectionPool { paused_waiter: Arc, pub stats: Arc, + + /// AuthInfo + pub auth_hash: Arc>>, } impl ConnectionPool { @@ -237,6 +249,7 @@ impl ConnectionPool { // Sort by shard number to ensure consistency. shard_ids.sort_by_key(|k| k.parse::().unwrap()); + let pool_auth_hash: Arc>> = Arc::new(RwLock::new(None)); for shard_idx in &shard_ids { let shard = &pool_config.shards[shard_idx]; @@ -293,12 +306,35 @@ impl ConnectionPool { replica_number += 1; } + // We assume every server in the pool share user/passwords + let auth_passthrough = AuthPassthrough::from_pool_config(pool_config); + + if let Some(apt) = &auth_passthrough { + match apt.fetch_hash(&address).await { + Ok(ok) => { + if let Some(ref pool_auth_hash_value) = *(pool_auth_hash.read()) { + if ok != *pool_auth_hash_value { + warn!("Hash is not the same across shards of the same pool, client auth will \ + be done using last obtained hash. Server: {}:{}, Database: {}", server.host, server.port, shard.database); + } + } + debug!("Hash obtained for {:?}", address); + { + let mut pool_auth_hash = pool_auth_hash.write(); + *pool_auth_hash = Some(ok.clone()); + } + }, + Err(err) => warn!("Could not obtain password hashes using auth_query config, ignoring. Error: {:?}", err), + } + } + let manager = ServerPool::new( address.clone(), user.clone(), &shard.database, client_server_map.clone(), pool_stats.clone(), + pool_auth_hash.clone(), ); let connect_timeout = match pool_config.connect_timeout { @@ -330,6 +366,12 @@ impl ConnectionPool { } assert_eq!(shards.len(), addresses.len()); + if let Some(ref _auth_hash) = *(pool_auth_hash.clone().read()) { + info!( + "Auth hash obtained from query_auth for pool {{ name: {}, user: {} }}", + pool_name, user.username + ); + } let pool = ConnectionPool { databases: shards, @@ -338,6 +380,7 @@ impl ConnectionPool { banlist: Arc::new(RwLock::new(banlist)), config_hash: new_pool_hash_value, server_info: Arc::new(RwLock::new(BytesMut::new())), + auth_hash: pool_auth_hash, settings: PoolSettings { pool_mode: pool_config.pool_mode, load_balancing_mode: pool_config.load_balancing_mode, @@ -366,6 +409,9 @@ impl ConnectionPool { .clone() .map(|regex| Regex::new(regex.as_str()).unwrap()), regex_search_limit: pool_config.regex_search_limit.unwrap_or(1000), + auth_query: pool_config.auth_query.clone(), + auth_query_user: pool_config.auth_query_user.clone(), + auth_query_password: pool_config.auth_query_password.clone(), }, validated: Arc::new(AtomicBool::new(false)), paused: Arc::new(AtomicBool::new(false)), @@ -389,7 +435,8 @@ impl ConnectionPool { Ok(()) } - /// Connect to all shards and grab server information. + /// Connect to all shards, grab server information, and possibly + /// passwords to use in client auth. /// Return server information we will pass to the clients /// when they connect. /// This also warms up the pool for clients that connect when @@ -803,6 +850,7 @@ pub struct ServerPool { database: String, client_server_map: ClientServerMap, stats: Arc, + auth_hash: Arc>>, } impl ServerPool { @@ -812,6 +860,7 @@ impl ServerPool { database: &str, client_server_map: ClientServerMap, stats: Arc, + auth_hash: Arc>>, ) -> ServerPool { ServerPool { address, @@ -819,6 +868,7 @@ impl ServerPool { database: database.to_string(), client_server_map, stats, + auth_hash, } } } @@ -847,6 +897,7 @@ impl ManageConnection for ServerPool { &self.database, self.client_server_map.clone(), stats.clone(), + self.auth_hash.clone(), ) .await { diff --git a/src/query_router.rs b/src/query_router.rs index 578c73901..0ea907b56 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1110,6 +1110,9 @@ mod test { sharding_key_regex: None, shard_id_regex: None, regex_search_limit: 1000, + auth_query: None, + auth_query_password: None, + auth_query_user: None, }; let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); @@ -1171,6 +1174,9 @@ mod test { sharding_key_regex: Some(Regex::new(r"/\* sharding_key: (\d+) \*/").unwrap()), shard_id_regex: Some(Regex::new(r"/\* shard_id: (\d+) \*/").unwrap()), regex_search_limit: 1000, + auth_query: None, + auth_query_password: None, + auth_query_user: None, }; let mut qr = QueryRouter::new(); qr.update_pool_settings(pool_settings.clone()); diff --git a/src/server.rs b/src/server.rs index d09313ec8..37f0e0c7b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,7 +1,11 @@ /// Implementation of the PostgreSQL server (database) protocol. /// Here we are pretending to the a Postgres client. use bytes::{Buf, BufMut, BytesMut}; +use fallible_iterator::FallibleIterator; use log::{debug, error, info, trace, warn}; +use parking_lot::{Mutex, RwLock}; +use postgres_protocol::message; +use std::collections::HashMap; use std::io::Read; use std::sync::Arc; use std::time::SystemTime; @@ -81,6 +85,7 @@ impl Server { database: &str, client_server_map: ClientServerMap, stats: Arc, + auth_hash: Arc>>, ) -> Result { let mut stream = match TcpStream::connect(&format!("{}:{}", &address.host, address.port)).await { @@ -106,7 +111,10 @@ impl Server { // We'll be handling multiple packets, but they will all be structured the same. // We'll loop here until this exchange is complete. - let mut scram = ScramSha256::new(&user.password); + let mut scram: Option = None; + if let Some(password) = &user.password.clone() { + scram = Some(ScramSha256::new(password)); + } loop { let code = match stream.read_u8().await { @@ -143,13 +151,40 @@ impl Server { Err(_) => return Err(Error::SocketError(format!("Error reading salt on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; - md5_password(&mut stream, &user.username, &user.password, &salt[..]) - .await?; + match &user.password { + // Using plaintext password + Some(password) => { + md5_password(&mut stream, &user.username, password, &salt[..]) + .await? + } + + // Using auth passthrough, in this case we should already have a + // hash obtained when the pool was validated. If we reach this point + // and don't have a hash, we return an error. + None => { + let option_hash = (*auth_hash.read()).clone(); + match option_hash { + Some(hash) => + md5_password_with_hash( + &mut stream, + &hash, + &salt[..], + ) + .await?, + None => + return Err(Error::AuthError(format!("Auth passthrough (auth_query) failed and no user password is set in cleartext for {{ username: {:?}, database: {:?} }}", user.username, database))) + } + } + } } AUTHENTICATION_SUCCESSFUL => (), SASL => { + if scram.is_none() { + return Err(Error::AuthError(format!("SASL auth required and not password specified, auth passthrough (auth_query) method is currently unsupported for SASL auth {{ username: {:?}, database: {:?} }}", user.username, database))); + } + debug!("Starting SASL authentication"); let sasl_len = (len - 8) as usize; let mut sasl_auth = vec![0u8; sasl_len]; @@ -165,7 +200,7 @@ impl Server { debug!("Using {}", SCRAM_SHA_256); // Generate client message. - let sasl_response = scram.message(); + let sasl_response = scram.as_mut().unwrap().message(); // SASLInitialResponse (F) let mut res = BytesMut::new(); @@ -202,7 +237,7 @@ impl Server { }; let msg = BytesMut::from(&sasl_data[..]); - let sasl_response = scram.update(&msg)?; + let sasl_response = scram.as_mut().unwrap().update(&msg)?; // SASLResponse let mut res = BytesMut::new(); @@ -222,7 +257,11 @@ impl Server { Err(_) => return Err(Error::SocketError(format!("Error reading sasl final message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), }; - match scram.finish(&BytesMut::from(&sasl_final[..])) { + match scram + .as_mut() + .unwrap() + .finish(&BytesMut::from(&sasl_final[..])) + { Ok(_) => { debug!("SASL authentication successful"); } @@ -696,6 +735,105 @@ impl Server { None => (), } } + + // This is so we can execute out of band queries to the server. + // The connection will be opened, the query executed and closed. + pub async fn exec_simple_query( + address: &Address, + user: &User, + query: &str, + ) -> Result, Error> { + let client_server_map: ClientServerMap = Arc::new(Mutex::new(HashMap::new())); + + debug!("Connecting to server to obtain auth hashes."); + let mut server = Server::startup( + address, + user, + &address.database, + client_server_map, + Arc::new(ServerStats::default()), + Arc::new(RwLock::new(None)), + ) + .await?; + debug!("Connected!, sending query."); + server.send(&simple_query(query)).await?; + let mut message = server.recv().await?; + + Ok(parse_query_message(&mut message).await?) + } +} + +async fn parse_query_message(message: &mut BytesMut) -> Result, Error> { + let mut pair = Vec::::new(); + match message::backend::Message::parse(message) { + Ok(Some(message::backend::Message::RowDescription(_description))) => {} + Ok(Some(message::backend::Message::ErrorResponse(err))) => { + return Err(Error::ProtocolSyncError(format!( + "Protocol error parsing response. Err: {:?}", + err.fields() + .iterator() + .fold(String::default(), |acc, element| acc + + element.unwrap().value()) + ))) + } + Ok(_) => { + return Err(Error::ProtocolSyncError( + "Protocol error, expected Row Description.".to_string(), + )) + } + Err(err) => { + return Err(Error::ProtocolSyncError(format!( + "Protocol error parsing response. Err: {:?}", + err + ))) + } + } + + while !message.is_empty() { + match message::backend::Message::parse(message) { + Ok(postgres_message) => { + match postgres_message { + Some(message::backend::Message::DataRow(data)) => { + let buf = data.buffer(); + trace!("Data: {:?}", buf); + + for item in data.ranges().iterator() { + match item.as_ref() { + Ok(range) => match range { + Some(range) => { + pair.push(String::from_utf8_lossy(&buf[range.clone()]).to_string()); + } + None => return Err(Error::ProtocolSyncError(String::from( + "Data expected while receiving query auth data, found nothing.", + ))), + }, + Err(err) => { + return Err(Error::ProtocolSyncError(format!( + "Data error, err: {:?}", + err + ))) + } + } + } + } + Some(message::backend::Message::CommandComplete(_)) => {} + Some(message::backend::Message::ReadyForQuery(_)) => {} + _ => { + return Err(Error::ProtocolSyncError( + "Unexpected message while receiving auth query data.".to_string(), + )) + } + } + } + Err(err) => { + return Err(Error::ProtocolSyncError(format!( + "Parse error, err: {:?}", + err + ))) + } + }; + } + Ok(pair) } impl Drop for Server { diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index e57d85298..93e945502 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -36,6 +36,15 @@ services: POSTGRES_PASSWORD: postgres POSTGRES_INITDB_ARGS: --auth-local=scram-sha-256 --auth-host=scram-sha-256 --auth=scram-sha-256 command: ["postgres", "-p", "9432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] + pg5: + image: postgres:14 + network_mode: "service:main" + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-p", "10432"] main: build: . command: ["bash", "/app/tests/docker/run.sh"] diff --git a/tests/ruby/auth_query_spec.rb b/tests/ruby/auth_query_spec.rb new file mode 100644 index 000000000..1ac621648 --- /dev/null +++ b/tests/ruby/auth_query_spec.rb @@ -0,0 +1,215 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' +require_relative 'helpers/auth_query_helper' + +describe "Auth Query" do + let(:configured_instances) {[5432, 10432]} + let(:config_user) { { 'username' => 'sharding_user', 'password' => 'sharding_user' } } + let(:pg_user) { { 'username' => 'sharding_user', 'password' => 'sharding_user' } } + let(:processes) { Helpers::AuthQuery.single_shard_auth_query(pool_name: "sharded_db", pg_user: pg_user, config_user: config_user, extra_conf: config, wait_until_ready: wait_until_ready ) } + let(:config) { {} } + let(:wait_until_ready) { true } + + after do + unless @failing_process + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + @failing_process = false + end + + context "when auth_query is not configured" do + context 'and cleartext passwords are set' do + it "uses local passwords" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", config_user['username'], config_user['password'])) + + expect(conn.async_exec("SELECT 1 + 2")).not_to be_nil + end + end + + context 'and cleartext passwords are not set' do + let(:config_user) { { 'username' => 'sharding_user' } } + + it "does not start because it is not possible to authenticate" do + @failing_process = true + expect { processes.pgcat }.to raise_error(StandardError, /You have to specify a user password for every pool if auth_query is not specified/) + end + end + end + + context 'when auth_query is configured' do + context 'with global configuration' do + around(:example) do |example| + + # Set up auth query + Helpers::AuthQuery.set_up_auth_query_for_user( + user: 'md5_auth_user', + password: 'secret' + ); + + example.run + + # Drop auth query support + Helpers::AuthQuery.tear_down_auth_query_for_user( + user: 'md5_auth_user', + password: 'secret' + ); + end + + context 'with correct global parameters' do + let(:config) { { 'general' => { 'auth_query' => "SELECT * FROM public.user_lookup('$1');", 'auth_query_user' => 'md5_auth_user', 'auth_query_password' => 'secret' } } } + context 'and with cleartext passwords set' do + it 'it uses local passwords' do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", pg_user['username'], pg_user['password'])) + expect(conn.exec("SELECT 1 + 2")).not_to be_nil + end + end + + context 'and with cleartext passwords not set' do + let(:config_user) { { 'username' => 'sharding_user', 'password' => 'sharding_user' } } + + it 'it uses obtained passwords' do + connection_string = processes.pgcat.connection_string("sharded_db", pg_user['username'], pg_user['password']) + conn = PG.connect(connection_string) + expect(conn.async_exec("SELECT 1 + 2")).not_to be_nil + end + + it 'allows passwords to be changed without closing existing connections' do + pgconn = PG.connect(processes.pgcat.connection_string("sharded_db", pg_user['username'])) + expect(pgconn.exec("SELECT 1 + 2")).not_to be_nil + Helpers::AuthQuery.exec_in_instances(query: "ALTER USER #{pg_user['username']} WITH ENCRYPTED PASSWORD 'secret2';") + expect(pgconn.exec("SELECT 1 + 4")).not_to be_nil + Helpers::AuthQuery.exec_in_instances(query: "ALTER USER #{pg_user['username']} WITH ENCRYPTED PASSWORD '#{pg_user['password']}';") + end + + it 'allows passwords to be changed and that new password is needed when reconnecting' do + pgconn = PG.connect(processes.pgcat.connection_string("sharded_db", pg_user['username'])) + expect(pgconn.exec("SELECT 1 + 2")).not_to be_nil + Helpers::AuthQuery.exec_in_instances(query: "ALTER USER #{pg_user['username']} WITH ENCRYPTED PASSWORD 'secret2';") + newconn = PG.connect(processes.pgcat.connection_string("sharded_db", pg_user['username'], 'secret2')) + expect(newconn.exec("SELECT 1 + 2")).not_to be_nil + Helpers::AuthQuery.exec_in_instances(query: "ALTER USER #{pg_user['username']} WITH ENCRYPTED PASSWORD '#{pg_user['password']}';") + end + end + end + + context 'with wrong parameters' do + let(:config) { { 'general' => { 'auth_query' => 'SELECT 1', 'auth_query_user' => 'wrong_user', 'auth_query_password' => 'wrong' } } } + + context 'and with clear text passwords set' do + it "it uses local passwords" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", pg_user['username'], pg_user['password'])) + + expect(conn.async_exec("SELECT 1 + 2")).not_to be_nil + end + end + + context 'and with cleartext passwords not set' do + let(:config_user) { { 'username' => 'sharding_user' } } + it "it fails to start as it cannot authenticate against servers" do + @failing_process = true + expect { PG.connect(processes.pgcat.connection_string("sharded_db", pg_user['username'], pg_user['password'])) }.to raise_error(StandardError, /Error trying to obtain password from auth_query/ ) + end + + context 'and we fix the issue and reload' do + let(:wait_until_ready) { false } + + it 'fails in the beginning but starts working after reloading config' do + connection_string = processes.pgcat.connection_string("sharded_db", pg_user['username'], pg_user['password']) + while !(processes.pgcat.logs =~ /Waiting for clients/) do + sleep 0.5 + end + + expect { PG.connect(connection_string)}.to raise_error(PG::ConnectionBad) + expect(processes.pgcat.logs).to match(/Error trying to obtain password from auth_query/) + + current_config = processes.pgcat.current_config + config = { 'general' => { 'auth_query' => "SELECT * FROM public.user_lookup('$1');", 'auth_query_user' => 'md5_auth_user', 'auth_query_password' => 'secret' } } + processes.pgcat.update_config(current_config.deep_merge(config)) + processes.pgcat.reload_config + + conn = nil + expect { conn = PG.connect(connection_string)}.not_to raise_error + expect(conn.async_exec("SELECT 1 + 2")).not_to be_nil + end + end + end + end + end + + context 'with per pool configuration' do + around(:example) do |example| + + # Set up auth query + Helpers::AuthQuery.set_up_auth_query_for_user( + user: 'md5_auth_user', + password: 'secret' + ); + + Helpers::AuthQuery.set_up_auth_query_for_user( + user: 'md5_auth_user1', + password: 'secret', + database: 'shard1' + ); + + example.run + + # Tear down auth query + Helpers::AuthQuery.tear_down_auth_query_for_user( + user: 'md5_auth_user', + password: 'secret' + ); + + Helpers::AuthQuery.tear_down_auth_query_for_user( + user: 'md5_auth_user1', + password: 'secret', + database: 'shard1' + ); + end + + context 'with correct parameters' do + let(:processes) { Helpers::AuthQuery.two_pools_auth_query(pool_names: ["sharded_db0", "sharded_db1"], pg_user: pg_user, config_user: config_user, extra_conf: config ) } + let(:config) { + { 'pools' => + { + 'sharded_db0' => { + 'auth_query' => "SELECT * FROM public.user_lookup('$1');", + 'auth_query_user' => 'md5_auth_user', + 'auth_query_password' => 'secret' + }, + 'sharded_db1' => { + 'auth_query' => "SELECT * FROM public.user_lookup('$1');", + 'auth_query_user' => 'md5_auth_user1', + 'auth_query_password' => 'secret' + }, + } + } + } + + context 'and with cleartext passwords set' do + it 'it uses local passwords' do + conn = PG.connect(processes.pgcat.connection_string("sharded_db0", pg_user['username'], pg_user['password'])) + expect(conn.exec("SELECT 1 + 2")).not_to be_nil + conn = PG.connect(processes.pgcat.connection_string("sharded_db1", pg_user['username'], pg_user['password'])) + expect(conn.exec("SELECT 1 + 2")).not_to be_nil + end + end + + context 'and with cleartext passwords not set' do + let(:config_user) { { 'username' => 'sharding_user' } } + + it 'it uses obtained passwords' do + connection_string = processes.pgcat.connection_string("sharded_db0", pg_user['username'], pg_user['password']) + conn = PG.connect(connection_string) + expect(conn.async_exec("SELECT 1 + 2")).not_to be_nil + connection_string = processes.pgcat.connection_string("sharded_db1", pg_user['username'], pg_user['password']) + conn = PG.connect(connection_string) + expect(conn.async_exec("SELECT 1 + 2")).not_to be_nil + end + end + + end + end + end +end diff --git a/tests/ruby/helpers/auth_query_helper.rb b/tests/ruby/helpers/auth_query_helper.rb new file mode 100644 index 000000000..60e85713b --- /dev/null +++ b/tests/ruby/helpers/auth_query_helper.rb @@ -0,0 +1,173 @@ +module Helpers + module AuthQuery + def self.single_shard_auth_query( + pg_user:, + config_user:, + pool_name:, + extra_conf: {}, + log_level: 'debug', + wait_until_ready: true + ) + + user = { + "pool_size" => 10, + "statement_timeout" => 0, + } + + pgcat = PgcatProcess.new(log_level) + pgcat_cfg = pgcat.current_config.deep_merge(extra_conf) + + primary = PgInstance.new(5432, pg_user["username"], pg_user["password"], "shard0") + replica = PgInstance.new(10432, pg_user["username"], pg_user["password"], "shard0") + + # Main proxy configs + pgcat_cfg["pools"] = { + "#{pool_name}" => { + "default_role" => "any", + "pool_mode" => "transaction", + "load_balancing_mode" => "random", + "primary_reads_enabled" => false, + "query_parser_enabled" => false, + "sharding_function" => "pg_bigint_hash", + "shards" => { + "0" => { + "database" => "shard0", + "servers" => [ + ["localhost", primary.port.to_s, "primary"], + ["localhost", replica.port.to_s, "replica"], + ] + }, + }, + "users" => { "0" => user.merge(config_user) } + } + } + pgcat_cfg["general"]["port"] = pgcat.port + pgcat.update_config(pgcat_cfg) + pgcat.start + + pgcat.wait_until_ready( + pgcat.connection_string( + "sharded_db", + pg_user['username'], + pg_user['password'] + ) + ) if wait_until_ready + + OpenStruct.new.tap do |struct| + struct.pgcat = pgcat + struct.primary = primary + struct.replicas = [replica] + struct.all_databases = [primary] + end + end + + def self.two_pools_auth_query( + pg_user:, + config_user:, + pool_names:, + extra_conf: {}, + log_level: 'debug' + ) + + user = { + "pool_size" => 10, + "statement_timeout" => 0, + } + + pgcat = PgcatProcess.new(log_level) + pgcat_cfg = pgcat.current_config + + primary = PgInstance.new(5432, pg_user["username"], pg_user["password"], "shard0") + replica = PgInstance.new(10432, pg_user["username"], pg_user["password"], "shard0") + + pool_template = Proc.new do |database| + { + "default_role" => "any", + "pool_mode" => "transaction", + "load_balancing_mode" => "random", + "primary_reads_enabled" => false, + "query_parser_enabled" => false, + "sharding_function" => "pg_bigint_hash", + "shards" => { + "0" => { + "database" => database, + "servers" => [ + ["localhost", primary.port.to_s, "primary"], + ["localhost", replica.port.to_s, "replica"], + ] + }, + }, + "users" => { "0" => user.merge(config_user) } + } + end + # Main proxy configs + pgcat_cfg["pools"] = { + "#{pool_names[0]}" => pool_template.call("shard0"), + "#{pool_names[1]}" => pool_template.call("shard1") + } + + pgcat_cfg["general"]["port"] = pgcat.port + pgcat.update_config(pgcat_cfg.deep_merge(extra_conf)) + pgcat.start + + pgcat.wait_until_ready(pgcat.connection_string("sharded_db0", pg_user['username'], pg_user['password'])) + + OpenStruct.new.tap do |struct| + struct.pgcat = pgcat + struct.primary = primary + struct.replicas = [replica] + struct.all_databases = [primary] + end + end + + def self.create_query_auth_function(user) + return <<-SQL +CREATE OR REPLACE FUNCTION public.user_lookup(in i_username text, out uname text, out phash text) +RETURNS record AS $$ +BEGIN + SELECT usename, passwd FROM pg_catalog.pg_shadow + WHERE usename = i_username INTO uname, phash; + RETURN; +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +GRANT EXECUTE ON FUNCTION public.user_lookup(text) TO #{user}; +SQL + end + + def self.exec_in_instances(query:, instance_ports: [ 5432, 10432 ], database: 'postgres', user: 'postgres', password: 'postgres') + instance_ports.each do |port| + c = PG.connect("postgres://#{user}:#{password}@localhost:#{port}/#{database}") + c.exec(query) + c.close + end + end + + def self.set_up_auth_query_for_user(user:, password:, instance_ports: [ 5432, 10432 ], database: 'shard0' ) + instance_ports.each do |port| + connection = PG.connect("postgres://postgres:postgres@localhost:#{port}/#{database}") + connection.exec(self.drop_query_auth_function(user)) rescue PG::UndefinedFunction + connection.exec("DROP ROLE #{user}") rescue PG::UndefinedObject + connection.exec("CREATE ROLE #{user} ENCRYPTED PASSWORD '#{password}' LOGIN;") + connection.exec(self.create_query_auth_function(user)) + connection.close + end + end + + def self.tear_down_auth_query_for_user(user:, password:, instance_ports: [ 5432, 10432 ], database: 'shard0' ) + instance_ports.each do |port| + connection = PG.connect("postgres://postgres:postgres@localhost:#{port}/#{database}") + connection.exec(self.drop_query_auth_function(user)) rescue PG::UndefinedFunction + connection.exec("DROP ROLE #{user}") + connection.close + end + end + + def self.drop_query_auth_function(user) + return <<-SQL +REVOKE ALL ON FUNCTION public.user_lookup(text) FROM public, #{user}; +DROP FUNCTION public.user_lookup(in i_username text, out uname text, out phash text); +SQL + end + end +end diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index c4ebab7f4..13dc6686d 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -3,6 +3,13 @@ require_relative 'pgcat_process' require_relative 'pg_instance' +class ::Hash + def deep_merge(second) + merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 } + self.merge(second, &merger) + end +end + module Helpers module Pgcat def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random", log_level="info") diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index 6120c99f6..e1dbea8bf 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -67,17 +67,21 @@ def reload_config def start raise StandardError, "Process is already started" unless @pid.nil? @pid = Process.spawn(@env, @command, err: @log_filename, out: @log_filename) + Process.detach(@pid) ObjectSpace.define_finalizer(@log_filename, proc { PgcatProcess.finalize(@pid, @log_filename, @config_filename) }) return self end - def wait_until_ready + def wait_until_ready(connection_string = nil) exc = nil 10.times do - PG::connect(example_connection_string).close + Process.kill 0, @pid + PG::connect(connection_string || example_connection_string).close return self + rescue Errno::ESRCH + raise StandardError, "Process #{@pid} died. #{logs}" rescue => e exc = e sleep(0.5) @@ -108,13 +112,10 @@ def admin_connection_string "postgresql://#{username}:#{password}@0.0.0.0:#{@port}/pgcat" end - def connection_string(pool_name, username) + def connection_string(pool_name, username, password = nil) cfg = current_config - user_idx, user_obj = cfg["pools"][pool_name]["users"].detect { |k, user| user["username"] == username } - password = user_obj["password"] - - "postgresql://#{username}:#{password}@0.0.0.0:#{@port}/#{pool_name}" + "postgresql://#{username}:#{password || user_obj["password"]}@0.0.0.0:#{@port}/#{pool_name}" end def example_connection_string From d05ec62ed2a9c51384e3ab4b3558072b69d70c28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Mar 2023 09:34:51 -0700 Subject: [PATCH 371/551] chore(deps): bump postgres-protocol from 0.6.4 to 0.6.5 (#391) Bumps [postgres-protocol](https://github.com/sfackler/rust-postgres) from 0.6.4 to 0.6.5. - [Release notes](https://github.com/sfackler/rust-postgres/releases) - [Commits](https://github.com/sfackler/rust-postgres/compare/postgres-protocol-v0.6.4...postgres-protocol-v0.6.5) --- updated-dependencies: - dependency-name: postgres-protocol dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 16 +++++----------- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad7bb8ea9..9074e2e9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,12 +54,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.0" @@ -750,7 +744,7 @@ dependencies = [ "arc-swap", "async-trait", "atomic_enum", - "base64 0.21.0", + "base64", "bb8", "bytes", "chrono", @@ -840,11 +834,11 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "postgres-protocol" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "878c6cbf956e03af9aa8204b407b9cbf47c072164800aa918c516cd4b056c50c" +checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" dependencies = [ - "base64 0.13.1", + "base64", "byteorder", "bytes", "fallible-iterator", @@ -983,7 +977,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64 0.21.0", + "base64", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 89cfe6430..462c859fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ futures = "0.3" socket2 = { version = "0.4.7", features = ["all"] } nix = "0.26.2" atomic_enum = "0.2.0" -postgres-protocol = "0.6.4" +postgres-protocol = "0.6.5" fallible-iterator = "0.2" [target.'cfg(not(target_env = "msvc"))'.dependencies] From 1a491cc89974bcda9b83f6d0310d6dfedcc7331c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Mar 2023 09:35:01 -0700 Subject: [PATCH 372/551] chore(deps): bump futures from 0.3.27 to 0.3.28 (#392) Bumps [futures](https://github.com/rust-lang/futures-rs) from 0.3.27 to 0.3.28. - [Release notes](https://github.com/rust-lang/futures-rs/releases) - [Changelog](https://github.com/rust-lang/futures-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/futures-rs/compare/0.3.27...0.3.28) --- updated-dependencies: - dependency-name: futures dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9074e2e9f..3b5e3b808 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,9 +277,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -292,9 +292,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -302,15 +302,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -319,38 +319,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-macro" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.9", ] [[package]] name = "futures-sink" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-channel", "futures-core", From 56ad7bef97ebdd465f7c849b99523452b9d6d20c Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sat, 1 Apr 2023 15:27:57 -0500 Subject: [PATCH 373/551] Protocol-level test helpers (#393) I needed to have granular control over protocol message testing. For example, being able to send protocol messages one-by-one and then be able to inspect the results. In order to do that, I created this low-level ruby client that can be used to send protocol messages in any order without blocking and also allows inspection of response messages. --- tests/ruby/helpers/pg_socket.rb | 259 +++++++++++++++++++++++++++++ tests/ruby/helpers/pgcat_helper.rb | 1 + tests/ruby/protocol_spec.rb | 155 +++++++++++++++++ 3 files changed, 415 insertions(+) create mode 100644 tests/ruby/helpers/pg_socket.rb create mode 100644 tests/ruby/protocol_spec.rb diff --git a/tests/ruby/helpers/pg_socket.rb b/tests/ruby/helpers/pg_socket.rb new file mode 100644 index 000000000..422344912 --- /dev/null +++ b/tests/ruby/helpers/pg_socket.rb @@ -0,0 +1,259 @@ +require 'socket' +require 'digest/md5' + +BACKEND_MESSAGE_CODES = { + 'Z' => "ReadyForQuery", + 'C' => "CommandComplete", + 'T' => "RowDescription", + 'D' => "DataRow", + '1' => "ParseComplete", + '2' => "BindComplete", + 'E' => "ErrorResponse", + 's' => "PortalSuspended", +} + +class PostgresSocket + def initialize(host, port) + @port = port + @host = host + @socket = TCPSocket.new @host, @port + @parameters = {} + @verbose = true + end + + def send_md5_password_message(username, password, salt) + m = Digest::MD5.hexdigest(password + username) + m = Digest::MD5.hexdigest(m + salt.map(&:chr).join("")) + m = 'md5' + m + bytes = (m.split("").map(&:ord) + [0]).flatten + message_size = bytes.count + 4 + + message = [] + + message << 'p'.ord + message << [message_size].pack('l>').unpack('CCCC') # 4 + message << bytes + message.flatten! + + + @socket.write(message.pack('C*')) + end + + def send_startup_message(username, database, password) + message = [] + + message << [196608].pack('l>').unpack('CCCC') # 4 + message << "user".split('').map(&:ord) # 4, 8 + message << 0 # 1, 9 + message << username.split('').map(&:ord) # 2, 11 + message << 0 # 1, 12 + message << "database".split('').map(&:ord) # 8, 20 + message << 0 # 1, 21 + message << database.split('').map(&:ord) # 2, 23 + message << 0 # 1, 24 + message << 0 # 1, 25 + message.flatten! + + total_message_size = message.size + 4 + + message_len = [total_message_size].pack('l>').unpack('CCCC') + + @socket.write([message_len + message].flatten.pack('C*')) + + sleep 0.1 + + read_startup_response(username, password) + end + + def read_startup_response(username, password) + message_code, message_len = @socket.recv(5).unpack("al>") + while message_code == 'R' + auth_code = @socket.recv(4).unpack('l>').pop + case auth_code + when 5 # md5 + salt = @socket.recv(4).unpack('CCCC') + send_md5_password_message(username, password, salt) + message_code, message_len = @socket.recv(5).unpack("al>") + when 0 # trust + break + end + end + loop do + message_code, message_len = @socket.recv(5).unpack("al>") + if message_code == 'Z' + @socket.recv(1).unpack("a") # most likely I + break # We are good to go + end + if message_code == 'S' + actual_message = @socket.recv(message_len - 4).unpack("C*") + k,v = actual_message.pack('U*').split(/\x00/) + @parameters[k] = v + end + if message_code == 'K' + process_id, secret_key = @socket.recv(message_len - 4).unpack("l>l>") + @parameters["process_id"] = process_id + @parameters["secret_key"] = secret_key + end + end + return @parameters + end + + def cancel_query + socket = TCPSocket.new @host, @port + process_key = @parameters["process_id"] + secret_key = @parameters["secret_key"] + message = [] + message << [16].pack('l>').unpack('CCCC') # 4 + message << [80877102].pack('l>').unpack('CCCC') # 4 + message << [process_key.to_i].pack('l>').unpack('CCCC') # 4 + message << [secret_key.to_i].pack('l>').unpack('CCCC') # 4 + message.flatten! + socket.write(message.flatten.pack('C*')) + socket.close + log "[F] Sent CancelRequest message" + end + + def send_query_message(query) + query_size = query.length + message_size = 1 + 4 + query_size + message = [] + message << "Q".ord + message << [message_size].pack('l>').unpack('CCCC') # 4 + message << query.split('').map(&:ord) # 2, 11 + message << 0 # 1, 12 + message.flatten! + @socket.write(message.flatten.pack('C*')) + log "[F] Sent Q message (#{query})" + end + + def send_parse_message(query) + query_size = query.length + message_size = 2 + 2 + 4 + query_size + message = [] + message << "P".ord + message << [message_size].pack('l>').unpack('CCCC') # 4 + message << 0 # unnamed statement + message << query.split('').map(&:ord) # 2, 11 + message << 0 # 1, 12 + message << [0, 0] + message.flatten! + @socket.write(message.flatten.pack('C*')) + log "[F] Sent P message (#{query})" + end + + def send_bind_message + message = [] + message << "B".ord + message << [12].pack('l>').unpack('CCCC') # 4 + message << 0 # unnamed statement + message << 0 # unnamed statement + message << [0, 0] # 2 + message << [0, 0] # 2 + message << [0, 0] # 2 + message.flatten! + @socket.write(message.flatten.pack('C*')) + log "[F] Sent B message" + end + + def send_describe_message(mode) + message = [] + message << "D".ord + message << [6].pack('l>').unpack('CCCC') # 4 + message << mode.ord + message << 0 # unnamed statement + message.flatten! + @socket.write(message.flatten.pack('C*')) + log "[F] Sent D message" + end + + def send_execute_message(limit=0) + message = [] + message << "E".ord + message << [9].pack('l>').unpack('CCCC') # 4 + message << 0 # unnamed statement + message << [limit].pack('l>').unpack('CCCC') # 4 + message.flatten! + @socket.write(message.flatten.pack('C*')) + log "[F] Sent E message" + end + + def send_sync_message + message = [] + message << "S".ord + message << [4].pack('l>').unpack('CCCC') # 4 + message.flatten! + @socket.write(message.flatten.pack('C*')) + log "[F] Sent S message" + end + + def send_copydone_message + message = [] + message << "c".ord + message << [4].pack('l>').unpack('CCCC') # 4 + message.flatten! + @socket.write(message.flatten.pack('C*')) + log "[F] Sent c message" + end + + def send_copyfail_message + message = [] + message << "f".ord + message << [5].pack('l>').unpack('CCCC') # 4 + message << 0 + message.flatten! + @socket.write(message.flatten.pack('C*')) + log "[F] Sent f message" + end + + def send_flush_message + message = [] + message << "H".ord + message << [4].pack('l>').unpack('CCCC') # 4 + message.flatten! + @socket.write(message.flatten.pack('C*')) + log "[F] Sent H message" + end + + def read_from_server() + output_messages = [] + retry_count = 0 + message_code = nil + message_len = 0 + loop do + begin + message_code, message_len = @socket.recv_nonblock(5).unpack("al>") + rescue IO::WaitReadable + return output_messages if retry_count > 50 + + retry_count += 1 + sleep(0.01) + next + end + message = { + code: message_code, + len: message_len, + bytes: [] + } + log "[B] #{BACKEND_MESSAGE_CODES[message_code] || ('UnknownMessage(' + message_code + ')')}" + + actual_message_length = message_len - 4 + if actual_message_length > 0 + message[:bytes] = @socket.recv(message_len - 4).unpack("C*") + log "\t#{message[:bytes].join(",")}" + log "\t#{message[:bytes].map(&:chr).join(" ")}" + end + output_messages << message + return output_messages if message_code == 'Z' + end + end + + def log(msg) + return unless @verbose + + puts msg + end + + def close + @socket.close + end +end diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index 13dc6686d..ad4c32a47 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -2,6 +2,7 @@ require 'ostruct' require_relative 'pgcat_process' require_relative 'pg_instance' +require_relative 'pg_socket' class ::Hash def deep_merge(second) diff --git a/tests/ruby/protocol_spec.rb b/tests/ruby/protocol_spec.rb new file mode 100644 index 000000000..9737650b2 --- /dev/null +++ b/tests/ruby/protocol_spec.rb @@ -0,0 +1,155 @@ +# frozen_string_literal: true +require_relative 'spec_helper' + + +describe "Portocol handling" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 1, "session") } + let(:sequence) { [] } + let(:pgcat_socket) { PostgresSocket.new('localhost', processes.pgcat.port) } + let(:pgdb_socket) { PostgresSocket.new('localhost', processes.all_databases.first.port) } + + after do + pgdb_socket.close + pgcat_socket.close + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + + def run_comparison(sequence, socket_a, socket_b) + sequence.each do |msg, *args| + socket_a.send(msg, *args) + socket_b.send(msg, *args) + + compare_messages( + socket_a.read_from_server, + socket_b.read_from_server + ) + end + end + + def compare_messages(msg_arr0, msg_arr1) + if msg_arr0.count != msg_arr1.count + error_output = [] + + error_output << "#{msg_arr0.count} : #{msg_arr1.count}" + error_output << "PgCat Messages" + error_output += msg_arr0.map { |message| "\t#{message[:code]} - #{message[:bytes].map(&:chr).join(" ")}" } + error_output << "PgServer Messages" + error_output += msg_arr1.map { |message| "\t#{message[:code]} - #{message[:bytes].map(&:chr).join(" ")}" } + error_desc = error_output.join("\n") + raise StandardError, "Message count mismatch #{error_desc}" + end + + (0..msg_arr0.count - 1).all? do |i| + msg0 = msg_arr0[i] + msg1 = msg_arr1[i] + + result = [ + msg0[:code] == msg1[:code], + msg0[:len] == msg1[:len], + msg0[:bytes] == msg1[:bytes], + ].all? + + next result if result + + if result == false + error_string = [] + if msg0[:code] != msg1[:code] + error_string << "code #{msg0[:code]} != #{msg1[:code]}" + end + if msg0[:len] != msg1[:len] + error_string << "len #{msg0[:len]} != #{msg1[:len]}" + end + if msg0[:bytes] != msg1[:bytes] + error_string << "bytes #{msg0[:bytes]} != #{msg1[:bytes]}" + end + err = error_string.join("\n") + + raise StandardError, "Message mismatch #{err}" + end + end + end + + RSpec.shared_examples "at parity with database" do + before do + pgcat_socket.send_startup_message("sharding_user", "sharded_db", "sharding_user") + pgdb_socket.send_startup_message("sharding_user", "shard0", "sharding_user") + end + + it "works" do + run_comparison(sequence, pgcat_socket, pgdb_socket) + end + end + + context "Cancel Query" do + let(:sequence) { + [ + [:send_query_message, "SELECT pg_sleep(5)"], + [:cancel_query] + ] + } + + it_behaves_like "at parity with database" + end + + xcontext "Simple query after parse" do + let(:sequence) { + [ + [:send_parse_message, "SELECT 5"], + [:send_query_message, "SELECT 1"], + [:send_bind_message], + [:send_describe_message, "P"], + [:send_execute_message], + [:send_sync_message], + ] + } + + # Known to fail due to PgCat not supporting flush + it_behaves_like "at parity with database" + end + + xcontext "Flush message" do + let(:sequence) { + [ + [:send_parse_message, "SELECT 1"], + [:send_flush_message] + ] + } + + # Known to fail due to PgCat not supporting flush + it_behaves_like "at parity with database" + end + + xcontext "Bind without parse" do + let(:sequence) { + [ + [:send_bind_message] + ] + } + # This is known to fail. + # Server responds immediately, Proxy buffers the message + it_behaves_like "at parity with database" + end + + context "Simple message" do + let(:sequence) { + [[:send_query_message, "SELECT 1"]] + } + + it_behaves_like "at parity with database" + end + + context "Extended protocol" do + let(:sequence) { + [ + [:send_parse_message, "SELECT 1"], + [:send_bind_message], + [:send_describe_message, "P"], + [:send_execute_message], + [:send_sync_message], + ] + } + + it_behaves_like "at parity with database" + end +end From 600b3ec2ff4141b7f3572ba2aa8dd751061fd0a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 2 Apr 2023 23:00:09 -0700 Subject: [PATCH 374/551] chore(deps): bump tokio-rustls from 0.23.4 to 0.24.0 (#394) Bumps [tokio-rustls](https://github.com/tokio-rs/tls) from 0.23.4 to 0.24.0. - [Release notes](https://github.com/tokio-rs/tls/releases) - [Commits](https://github.com/tokio-rs/tls/commits) --- updated-dependencies: - dependency-name: tokio-rustls dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 31 +++++++++++++++---------------- Cargo.toml | 2 +- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b5e3b808..be4cb3ecb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,14 +961,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "07180898a28ed6a7f7ba2311594308f595e3dd2e3c3812fa0a80a47b45f17e5d" dependencies = [ "log", "ring", + "rustls-webpki", "sct", - "webpki", ] [[package]] @@ -980,6 +980,16 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -1217,13 +1227,12 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" dependencies = [ "rustls", "tokio", - "webpki", ] [[package]] @@ -1437,16 +1446,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 462c859fc..02c80881f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ hmac = "0.12" sha2 = "0.10" base64 = "0.21" stringprep = "0.1" -tokio-rustls = "0.23" +tokio-rustls = "0.24" rustls-pemfile = "1" hyper = { version = "0.14", features = ["full"] } phf = { version = "0.11.1", features = ["macros"] } From 4b36452c0bacf952f0e975fcb410edf36a3d89ee Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 5 Apr 2023 15:06:19 -0700 Subject: [PATCH 375/551] Fix port; add user pool mode (#395) * Fix port; add user pool mode * will probably break our session/transaction mode tests --- pgcat.toml | 1 + src/auth_passthrough.rs | 1 + src/config.rs | 23 ++++++++++++++++++----- src/pool.rs | 5 ++++- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/pgcat.toml b/pgcat.toml index 0d883a332..a6a4af505 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -134,6 +134,7 @@ password = "sharding_user" # is the sum of pool_size across all users. pool_size = 9 + # Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. # 0 means it is disabled. statement_timeout = 0 diff --git a/src/auth_passthrough.rs b/src/auth_passthrough.rs index b9f0e97fb..76483e598 100644 --- a/src/auth_passthrough.rs +++ b/src/auth_passthrough.rs @@ -73,6 +73,7 @@ impl AuthPassthrough { password: Some(self.password.clone()), pool_size: 1, statement_timeout: 0, + pool_mode: None, }; let user = &address.username; diff --git a/src/config.rs b/src/config.rs index 6545457c6..37494f67f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -179,6 +179,7 @@ pub struct User { pub username: String, pub password: Option, pub pool_size: u32, + pub pool_mode: Option, #[serde(default)] // 0 pub statement_timeout: u64, } @@ -190,6 +191,7 @@ impl Default for User { password: None, pool_size: 15, statement_timeout: 0, + pool_mode: None, } } } @@ -201,7 +203,7 @@ pub struct General { pub host: String, #[serde(default = "General::default_port")] - pub port: i16, + pub port: u16, pub enable_prometheus_exporter: Option, pub prometheus_exporter_port: i16, @@ -261,7 +263,7 @@ impl General { "0.0.0.0".into() } - pub fn default_port() -> i16 { + pub fn default_port() -> u16 { 5432 } @@ -356,6 +358,7 @@ pub enum PoolMode { #[serde(alias = "session", alias = "Session")] Session, } + impl ToString for PoolMode { fn to_string(&self) -> String { match *self { @@ -816,8 +819,9 @@ impl Config { .to_string() ); info!( - "[pool: {}] Pool mode: {:?}", - pool_name, pool_config.pool_mode + "[pool: {}] Default pool mode: {}", + pool_name, + pool_config.pool_mode.to_string() ); info!( "[pool: {}] Load Balancing mode: {:?}", @@ -868,7 +872,16 @@ impl Config { info!( "[pool: {}][user: {}] Statement timeout: {}", pool_name, user.1.username, user.1.statement_timeout - ) + ); + info!( + "[pool: {}][user: {}] Pool mode: {}", + pool_name, + user.1.username, + match user.1.pool_mode { + Some(pool_mode) => pool_mode.to_string(), + None => pool_config.pool_mode.to_string(), + } + ); } } } diff --git a/src/pool.rs b/src/pool.rs index e1ab7cb4e..7f8e41c0a 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -382,7 +382,10 @@ impl ConnectionPool { server_info: Arc::new(RwLock::new(BytesMut::new())), auth_hash: pool_auth_hash, settings: PoolSettings { - pool_mode: pool_config.pool_mode, + pool_mode: match user.pool_mode { + Some(pool_mode) => pool_mode, + None => pool_config.pool_mode, + }, load_balancing_mode: pool_config.load_balancing_mode, // shards: pool_config.shards.clone(), shards: shard_ids.len(), From b13eeb0e73ba4e2f9703a2f4070d272602262e64 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 10 Apr 2023 14:51:01 -0700 Subject: [PATCH 376/551] A couple things (#397) * Format cleanup * fmt * finally --- .circleci/pgcat.toml | 2 +- .editorconfig | 14 ++++ Cargo.lock | 2 +- Cargo.toml | 2 +- examples/docker/pgcat.toml | 3 - pgcat.toml | 2 +- src/auth_passthrough.rs | 48 ++++++++---- src/client.rs | 146 +++++++++++++++++++++++++------------ src/config.rs | 6 +- src/errors.rs | 102 +++++++++++++++++++++++++- src/main.rs | 25 ++++--- src/server.rs | 113 +++++++++++++++++++++++----- src/stats/server.rs | 1 - 13 files changed, 366 insertions(+), 100 deletions(-) create mode 100644 .editorconfig diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 0d47ed722..377680a0f 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -39,7 +39,7 @@ log_client_connections = false log_client_disconnections = false # Reload config automatically if it changes. -autoreload = true +autoreload = 15000 # TLS tls_certificate = ".circleci/server.cert" diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..d7a2758df --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +root = true + +[*] +trim_trailing_whitespace = true +insert_final_newline = true + +[*.rs] +indent_style = space +indent_size = 4 +max_line_length = 120 + +[*.toml] +indent_style = space +indent_size = 2 diff --git a/Cargo.lock b/Cargo.lock index be4cb3ecb..5d6c56f05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -739,7 +739,7 @@ dependencies = [ [[package]] name = "pgcat" -version = "1.0.0" +version = "1.0.1" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 02c80881f..4e33c645f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.0.0" +version = "1.0.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index c41c8cdd6..bfc4c2e21 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -38,9 +38,6 @@ log_client_connections = false # If we should log client disconnections log_client_disconnections = false -# Reload config automatically if it changes. -autoreload = false - # TLS # tls_certificate = "server.cert" # tls_private_key = "server.key" diff --git a/pgcat.toml b/pgcat.toml index a6a4af505..183391977 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -45,7 +45,7 @@ log_client_connections = false log_client_disconnections = false # When set to true, PgCat reloads configs if it detects a change in the config file. -autoreload = false +autoreload = 15000 # Number of worker threads the Runtime will use (4 by default). worker_threads = 5 diff --git a/src/auth_passthrough.rs b/src/auth_passthrough.rs index 76483e598..f313dead0 100644 --- a/src/auth_passthrough.rs +++ b/src/auth_passthrough.rs @@ -1,4 +1,5 @@ use crate::errors::Error; +use crate::pool::ConnectionPool; use crate::server::Server; use log::debug; @@ -78,19 +79,25 @@ impl AuthPassthrough { let user = &address.username; - debug!("Connecting to server to obtain auth hashes."); + debug!("Connecting to server to obtain auth hashes"); + let auth_query = self.query.replace("$1", user); + match Server::exec_simple_query(address, &auth_user, &auth_query).await { Ok(password_data) => { if password_data.len() == 2 && password_data.first().unwrap() == user { - if let Some(stripped_hash) = password_data.last().unwrap().to_string().strip_prefix("md5") { - Ok(stripped_hash.to_string()) - } - else { - Err(Error::AuthPassthroughError( - "Obtained hash from auth_query does not seem to be in md5 format.".to_string(), - )) - } + if let Some(stripped_hash) = password_data + .last() + .unwrap() + .to_string() + .strip_prefix("md5") { + Ok(stripped_hash.to_string()) + } + else { + Err(Error::AuthPassthroughError( + "Obtained hash from auth_query does not seem to be in md5 format.".to_string(), + )) + } } else { Err(Error::AuthPassthroughError( "Data obtained from query does not follow the scheme 'user','hash'." @@ -99,10 +106,25 @@ impl AuthPassthrough { } } Err(err) => { - Err(Error::AuthPassthroughError( - format!("Error trying to obtain password from auth_query, ignoring hash for user '{}'. Error: {:?}", - user, err))) + Err(Error::AuthPassthroughError( + format!("Error trying to obtain password from auth_query, ignoring hash for user '{}'. Error: {:?}", + user, err)) + ) } - } + } + } +} + +pub async fn refetch_auth_hash(pool: &ConnectionPool) -> Result { + let address = pool.address(0, 0); + if let Some(apt) = AuthPassthrough::from_pool_settings(&pool.settings) { + let hash = apt.fetch_hash(address).await?; + + return Ok(hash); } + + Err(Error::ClientError(format!( + "Could not obtain hash for {{ username: {:?}, database: {:?} }}. Auth passthrough not enabled.", + address.username, address.database + ))) } diff --git a/src/client.rs b/src/client.rs index d75c069dd..4114e4bb2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,4 +1,4 @@ -use crate::errors::Error; +use crate::errors::{ClientIdentifier, Error}; use crate::pool::BanReason; /// Handle clients by pretending to be a PostgreSQL server. use bytes::{Buf, BufMut, BytesMut}; @@ -12,7 +12,7 @@ use tokio::sync::broadcast::Receiver; use tokio::sync::mpsc::Sender; use crate::admin::{generate_server_info_for_admin, handle_admin}; -use crate::auth_passthrough::AuthPassthrough; +use crate::auth_passthrough::refetch_auth_hash; use crate::config::{get_config, get_idle_client_in_transaction_timeout, Address, PoolMode}; use crate::constants::*; use crate::messages::*; @@ -202,7 +202,7 @@ pub async fn client_entrypoint( // Client probably disconnected rejecting our plain text connection. Ok((ClientConnectionType::Tls, _)) | Ok((ClientConnectionType::CancelQuery, _)) => Err(Error::ProtocolSyncError( - format!("Bad postgres client (plain)"), + "Bad postgres client (plain)".into(), )), Err(err) => Err(err), @@ -369,28 +369,14 @@ pub async fn startup_tls( } // Bad Postgres client. - Ok((ClientConnectionType::Tls, _)) | Ok((ClientConnectionType::CancelQuery, _)) => Err( - Error::ProtocolSyncError(format!("Bad postgres client (tls)")), - ), + Ok((ClientConnectionType::Tls, _)) | Ok((ClientConnectionType::CancelQuery, _)) => { + Err(Error::ProtocolSyncError("Bad postgres client (tls)".into())) + } Err(err) => Err(err), } } -async fn refetch_auth_hash(pool: &ConnectionPool) -> Result { - let address = pool.address(0, 0); - if let Some(apt) = AuthPassthrough::from_pool_settings(&pool.settings) { - let hash = apt.fetch_hash(address).await?; - - return Ok(hash); - } - - Err(Error::ClientError(format!( - "Could not obtain hash for {{ username: {:?}, database: {:?} }}. Auth passthrough not enabled.", - address.username, address.database - ))) -} - impl Client where S: tokio::io::AsyncRead + std::marker::Unpin, @@ -418,7 +404,7 @@ where Some(user) => user, None => { return Err(Error::ClientError( - "Missing user parameter on client startup".to_string(), + "Missing user parameter on client startup".into(), )) } }; @@ -433,6 +419,8 @@ where None => "pgcat", }; + let client_identifier = ClientIdentifier::new(&application_name, &username, &pool_name); + let admin = ["pgcat", "pgbouncer"] .iter() .filter(|db| *db == pool_name) @@ -463,7 +451,12 @@ where let code = match read.read_u8().await { Ok(p) => p, - Err(_) => return Err(Error::SocketError(format!("Error reading password code from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))), + Err(_) => { + return Err(Error::ClientSocketError( + "password code".into(), + client_identifier, + )) + } }; // PasswordMessage @@ -476,19 +469,30 @@ where let len = match read.read_i32().await { Ok(len) => len, - Err(_) => return Err(Error::SocketError(format!("Error reading password message length from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))), + Err(_) => { + return Err(Error::ClientSocketError( + "password message length".into(), + client_identifier, + )) + } }; let mut password_response = vec![0u8; (len - 4) as usize]; match read.read_exact(&mut password_response).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError(format!("Error reading password message from client {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))), + Err(_) => { + return Err(Error::ClientSocketError( + "password message".into(), + client_identifier, + )) + } }; // Authenticate admin user. let (transaction_mode, server_info) = if admin { let config = get_config(); + // Compare server and client hashes. let password_hash = md5_hash_password( &config.general.admin_username, @@ -497,10 +501,12 @@ where ); if password_hash != password_response { - warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name); + let error = Error::ClientGeneralError("Invalid password".into(), client_identifier); + + warn!("{}", error); wrong_password(&mut write, username).await?; - return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))); + return Err(error); } (false, generate_server_info_for_admin()) @@ -519,7 +525,10 @@ where ) .await?; - return Err(Error::ClientError(format!("Invalid pool name {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))); + return Err(Error::ClientGeneralError( + "Invalid pool name".into(), + client_identifier, + )); } }; @@ -530,16 +539,23 @@ where Some(md5_hash_password(username, password, &salt)) } else { if !get_config().is_auth_query_configured() { - return Err(Error::ClientError(format!("Client auth not possible, no cleartext password set for username: {:?} in config and auth passthrough (query_auth) is not set up.", username))); + return Err(Error::ClientAuthImpossible(username.into())); } let mut hash = (*pool.auth_hash.read()).clone(); if hash.is_none() { - warn!("Query auth configured but no hash password found for pool {}. Will try to refetch it.", pool_name); + warn!( + "Query auth configured \ + but no hash password found \ + for pool {}. Will try to refetch it.", + pool_name + ); + match refetch_auth_hash(&pool).await { Ok(fetched_hash) => { - warn!("Password for {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}, obtained. Updating.", username, pool_name, application_name); + warn!("Password for {}, obtained. Updating.", client_identifier); + { let mut pool_auth_hash = pool.auth_hash.write(); *pool_auth_hash = Some(fetched_hash.clone()); @@ -547,16 +563,12 @@ where hash = Some(fetched_hash); } + Err(err) => { - return Err( - Error::ClientError( - format!("No cleartext password set, and no auth passthrough could not obtain the hash from server for {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}, the error was: {:?}", - username, - pool_name, - application_name, - err) - ) - ); + return Err(Error::ClientAuthPassthroughError( + err.to_string(), + client_identifier, + )); } } }; @@ -570,20 +582,31 @@ where // // @TODO: we could end up fetching again the same password twice (see above). if password_hash.unwrap() != password_response { - warn!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}, will try to refetch it.", username, pool_name, application_name); + warn!( + "Invalid password {}, will try to refetch it.", + client_identifier + ); + let fetched_hash = refetch_auth_hash(&pool).await?; let new_password_hash = md5_hash_second_pass(&fetched_hash, &salt); // Ok password changed in server an auth is possible. if new_password_hash == password_response { - warn!("Password for {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}, changed in server. Updating.", username, pool_name, application_name); + warn!( + "Password for {}, changed in server. Updating.", + client_identifier + ); + { let mut pool_auth_hash = pool.auth_hash.write(); *pool_auth_hash = Some(fetched_hash); } } else { wrong_password(&mut write, username).await?; - return Err(Error::ClientError(format!("Invalid password {{ username: {:?}, pool_name: {:?}, application_name: {:?} }}", username, pool_name, application_name))); + return Err(Error::ClientGeneralError( + "Invalid password".into(), + client_identifier, + )); } } @@ -753,9 +776,9 @@ where &mut self.write, "terminating connection due to administrator command" ).await?; - self.stats.disconnect(); - return Ok(()) + self.stats.disconnect(); + return Ok(()); } // Admin clients ignore shutdown. @@ -928,11 +951,26 @@ where error!("Got Sync message but failed to get a connection from the pool"); self.buffer.clear(); } + error_response(&mut self.write, "could not get connection from the pool") .await?; - error!("Could not get connection from pool: {{ pool_name: {:?}, username: {:?}, shard: {:?}, role: \"{:?}\", error: \"{:?}\" }}", - self.pool_name.clone(), self.username.clone(), query_router.shard(), query_router.role(), err); + error!( + "Could not get connection from pool: \ + {{ \ + pool_name: {:?}, \ + username: {:?}, \ + shard: {:?}, \ + role: \"{:?}\", \ + error: \"{:?}\" \ + }}", + self.pool_name, + self.username, + query_router.shard(), + query_router.role(), + err + ); + continue; } }; @@ -999,11 +1037,25 @@ where Err(_) => { // Client idle in transaction timeout error_response(&mut self.write, "idle transaction timeout").await?; - error!("Client idle in transaction timeout: {{ pool_name: {:?}, username: {:?}, shard: {:?}, role: \"{:?}\"}}", self.pool_name.clone(), self.username.clone(), query_router.shard(), query_router.role()); + error!( + "Client idle in transaction timeout: \ + {{ \ + pool_name: {}, \ + username: {}, \ + shard: {}, \ + role: \"{:?}\" \ + }}", + self.pool_name, + self.username, + query_router.shard(), + query_router.role() + ); + break; } } } + Some(message) => { initial_message = None; message diff --git a/src/config.rs b/src/config.rs index 37494f67f..13528af22 100644 --- a/src/config.rs +++ b/src/config.rs @@ -245,8 +245,8 @@ pub struct General { #[serde(default = "General::default_worker_threads")] pub worker_threads: usize, - #[serde(default)] // False - pub autoreload: bool, + #[serde(default)] // None + pub autoreload: Option, pub tls_certificate: Option, pub tls_private_key: Option, @@ -335,7 +335,7 @@ impl Default for General { tcp_keepalives_interval: Self::default_tcp_keepalives_interval(), log_client_connections: false, log_client_disconnections: false, - autoreload: false, + autoreload: None, tls_certificate: None, tls_private_key: None, admin_username: String::from("admin"), diff --git a/src/errors.rs b/src/errors.rs index 58fc088b5..0930ab8bd 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,13 +1,19 @@ -/// Errors. +//! Errors. /// Various errors. #[derive(Debug, PartialEq)] pub enum Error { SocketError(String), + ClientSocketError(String, ClientIdentifier), + ClientGeneralError(String, ClientIdentifier), + ClientAuthImpossible(String), + ClientAuthPassthroughError(String, ClientIdentifier), ClientBadStartup, ProtocolSyncError(String), BadQuery(String), ServerError, + ServerStartupError(String, ServerIdentifier), + ServerAuthError(String, ServerIdentifier), BadConfig, AllServersDown, ClientError(String), @@ -18,3 +24,97 @@ pub enum Error { AuthError(String), AuthPassthroughError(String), } + +#[derive(Clone, PartialEq, Debug)] +pub struct ClientIdentifier { + pub application_name: String, + pub username: String, + pub pool_name: String, +} + +impl ClientIdentifier { + pub fn new(application_name: &str, username: &str, pool_name: &str) -> ClientIdentifier { + ClientIdentifier { + application_name: application_name.into(), + username: username.into(), + pool_name: pool_name.into(), + } + } +} + +impl std::fmt::Display for ClientIdentifier { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{{ application_name: {}, username: {}, pool_name: {} }}", + self.application_name, self.username, self.pool_name + ) + } +} + +#[derive(Clone, PartialEq, Debug)] +pub struct ServerIdentifier { + pub username: String, + pub database: String, +} + +impl ServerIdentifier { + pub fn new(username: &str, database: &str) -> ServerIdentifier { + ServerIdentifier { + username: username.into(), + database: database.into(), + } + } +} + +impl std::fmt::Display for ServerIdentifier { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{{ username: {}, database: {} }}", + self.username, self.database + ) + } +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match &self { + &Error::ClientSocketError(error, client_identifier) => write!( + f, + "Error reading {} from client {}", + error, client_identifier + ), + &Error::ClientGeneralError(error, client_identifier) => { + write!(f, "{} {}", error, client_identifier) + } + &Error::ClientAuthImpossible(username) => write!( + f, + "Client auth not possible, \ + no cleartext password set for username: {} \ + in config and auth passthrough (query_auth) \ + is not set up.", + username + ), + &Error::ClientAuthPassthroughError(error, client_identifier) => write!( + f, + "No cleartext password set, \ + and no auth passthrough could not \ + obtain the hash from server for {}, \ + the error was: {}", + client_identifier, error + ), + &Error::ServerStartupError(error, server_identifier) => write!( + f, + "Error reading {} on server startup {}", + error, server_identifier, + ), + &Error::ServerAuthError(error, server_identifier) => { + write!(f, "{} for {}", error, server_identifier,) + } + + // The rest can use Debug. + err => write!(f, "{:?}", err), + } + } +} diff --git a/src/main.rs b/src/main.rs index 4c8987f14..39c67c2ef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -179,16 +179,19 @@ fn main() -> Result<(), Box> { stats_collector.collect().await; }); - info!("Config autoreloader: {}", config.general.autoreload); + info!("Config autoreloader: {}", match config.general.autoreload { + Some(interval) => format!("{} ms", interval), + None => "disabled".into(), + }); - let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(15_000)); - let autoreload_client_server_map = client_server_map.clone(); + if let Some(interval) = config.general.autoreload { + let mut autoreload_interval = tokio::time::interval(tokio::time::Duration::from_millis(interval)); + let autoreload_client_server_map = client_server_map.clone(); - tokio::task::spawn(async move { - loop { - autoreload_interval.tick().await; - if config.general.autoreload { - info!("Automatically reloading config"); + tokio::task::spawn(async move { + loop { + autoreload_interval.tick().await; + debug!("Automatically reloading config"); if let Ok(changed) = reload_config(autoreload_client_server_map.clone()).await { if changed { @@ -196,8 +199,10 @@ fn main() -> Result<(), Box> { } }; } - } - }); + }); + }; + + #[cfg(windows)] let mut term_signal = win_signal::ctrl_close().unwrap(); diff --git a/src/server.rs b/src/server.rs index 37f0e0c7b..14862bd05 100644 --- a/src/server.rs +++ b/src/server.rs @@ -17,7 +17,7 @@ use tokio::net::{ use crate::config::{Address, User}; use crate::constants::*; -use crate::errors::Error; +use crate::errors::{Error, ServerIdentifier}; use crate::messages::*; use crate::mirrors::MirroringManager; use crate::pool::ClientServerMap; @@ -108,6 +108,7 @@ impl Server { let mut server_info = BytesMut::new(); let mut process_id: i32 = 0; let mut secret_key: i32 = 0; + let server_identifier = ServerIdentifier::new(&user.username, &database); // We'll be handling multiple packets, but they will all be structured the same. // We'll loop here until this exchange is complete. @@ -119,12 +120,22 @@ impl Server { loop { let code = match stream.read_u8().await { Ok(code) => code as char, - Err(_) => return Err(Error::SocketError(format!("Error reading message code on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "message code".into(), + server_identifier, + )) + } }; let len = match stream.read_i32().await { Ok(len) => len, - Err(_) => return Err(Error::SocketError(format!("Error reading message len on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "message len".into(), + server_identifier, + )) + } }; trace!("Message: {}", code); @@ -135,7 +146,12 @@ impl Server { // Determine which kind of authentication is required, if any. let auth_code = match stream.read_i32().await { Ok(auth_code) => auth_code, - Err(_) => return Err(Error::SocketError(format!("Error reading auth code on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "auth code".into(), + server_identifier, + )) + } }; trace!("Auth: {}", auth_code); @@ -148,7 +164,12 @@ impl Server { match stream.read_exact(&mut salt).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError(format!("Error reading salt on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "salt".into(), + server_identifier, + )) + } }; match &user.password { @@ -171,8 +192,12 @@ impl Server { &salt[..], ) .await?, - None => - return Err(Error::AuthError(format!("Auth passthrough (auth_query) failed and no user password is set in cleartext for {{ username: {:?}, database: {:?} }}", user.username, database))) + None => return Err( + Error::ServerAuthError( + "Auth passthrough (auth_query) failed and no user password is set in cleartext".into(), + server_identifier + ) + ), } } } @@ -182,16 +207,28 @@ impl Server { SASL => { if scram.is_none() { - return Err(Error::AuthError(format!("SASL auth required and not password specified, auth passthrough (auth_query) method is currently unsupported for SASL auth {{ username: {:?}, database: {:?} }}", user.username, database))); + return Err(Error::ServerAuthError( + "SASL auth required and no password specified. \ + Auth passthrough (auth_query) method is currently \ + unsupported for SASL auth" + .into(), + server_identifier, + )); } debug!("Starting SASL authentication"); + let sasl_len = (len - 8) as usize; let mut sasl_auth = vec![0u8; sasl_len]; match stream.read_exact(&mut sasl_auth).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError(format!("Error reading sasl message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "sasl message".into(), + server_identifier, + )) + } }; let sasl_type = String::from_utf8_lossy(&sasl_auth[..sasl_len - 2]); @@ -233,7 +270,12 @@ impl Server { match stream.read_exact(&mut sasl_data).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError(format!("Error reading sasl cont message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "sasl cont message".into(), + server_identifier, + )) + } }; let msg = BytesMut::from(&sasl_data[..]); @@ -254,7 +296,12 @@ impl Server { let mut sasl_final = vec![0u8; len as usize - 8]; match stream.read_exact(&mut sasl_final).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError(format!("Error reading sasl final message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "sasl final message".into(), + server_identifier, + )) + } }; match scram @@ -284,7 +331,12 @@ impl Server { 'E' => { let error_code = match stream.read_u8().await { Ok(error_code) => error_code, - Err(_) => return Err(Error::SocketError(format!("Error reading error code message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "error code message".into(), + server_identifier, + )) + } }; trace!("Error: {}", error_code); @@ -300,7 +352,12 @@ impl Server { match stream.read_exact(&mut error).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError(format!("Error reading error message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "error message".into(), + server_identifier, + )) + } }; // TODO: the error message contains multiple fields; we can decode them and @@ -319,7 +376,12 @@ impl Server { match stream.read_exact(&mut param).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError(format!("Error reading parameter status message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "parameter status message".into(), + server_identifier, + )) + } }; // Save the parameter so we can pass it to the client later. @@ -336,12 +398,22 @@ impl Server { // See: . process_id = match stream.read_i32().await { Ok(id) => id, - Err(_) => return Err(Error::SocketError(format!("Error reading process id message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "process id message".into(), + server_identifier, + )) + } }; secret_key = match stream.read_i32().await { Ok(id) => id, - Err(_) => return Err(Error::SocketError(format!("Error reading secret key message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "secret key message".into(), + server_identifier, + )) + } }; } @@ -351,7 +423,12 @@ impl Server { match stream.read_exact(&mut idle).await { Ok(_) => (), - Err(_) => return Err(Error::SocketError(format!("Error reading transaction status message on server startup {{ username: {:?}, database: {:?} }}", user.username, database))), + Err(_) => { + return Err(Error::ServerStartupError( + "transaction status message".into(), + server_identifier, + )) + } }; let (read, write) = stream.into_split(); @@ -413,7 +490,7 @@ impl Server { Ok(stream) => stream, Err(err) => { error!("Could not connect to server: {}", err); - return Err(Error::SocketError(format!("Error reading cancel message"))); + return Err(Error::SocketError("Error reading cancel message".into())); } }; configure_socket(&stream); diff --git a/src/stats/server.rs b/src/stats/server.rs index 009e9b57d..08968a12b 100644 --- a/src/stats/server.rs +++ b/src/stats/server.rs @@ -100,7 +100,6 @@ impl ServerStats { .server_idle(self.state.load(Ordering::Relaxed)); self.state.store(ServerState::Idle, Ordering::Relaxed); - self.set_undefined_application(); } /// Reports a server connection is disconecting from the pooler. From 5fc8b52a01d8f1480069e9148648e9059ce8731f Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Tue, 11 Apr 2023 09:37:16 +0800 Subject: [PATCH 377/551] Fix typos (#398) Found via `typos --format brief` --- CONFIG.md | 2 +- README.md | 2 +- examples/docker/pgcat.toml | 2 +- pgcat.toml | 4 ++-- src/config.rs | 4 ++-- src/messages.rs | 2 +- src/multi_logger.rs | 2 +- src/query_router.rs | 2 +- src/stats.rs | 4 ++-- src/stats/client.rs | 4 ++-- src/stats/server.rs | 2 +- tests/ruby/admin_spec.rb | 6 +++--- tests/ruby/load_balancing_spec.rb | 2 +- tests/ruby/sharding_spec.rb | 2 +- 14 files changed, 20 insertions(+), 20 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 3cec25301..667daf12c 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -152,7 +152,7 @@ default: example: "server.cert" ``` -Path to TLS Certficate file to use for TLS connections +Path to TLS Certificate file to use for TLS connections ### tls_private_key ``` diff --git a/README.md b/README.md index 63b5ab1a6..56a860573 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ You can open a Docker development environment where you can debug tests easier. ./dev/script/console ``` -This will open a terminal in an environment similar to that used in tests. In there, you can compile the pooler, run tests, do some debugging with the test environment, etc. Objects compiled inside the contaner (and bundled gems) will be placed in `dev/cache` so they don't interfere with what you have on your machine. +This will open a terminal in an environment similar to that used in tests. In there, you can compile the pooler, run tests, do some debugging with the test environment, etc. Objects compiled inside the container (and bundled gems) will be placed in `dev/cache` so they don't interfere with what you have on your machine. ## Usage diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index bfc4c2e21..5fd929de9 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -73,7 +73,7 @@ query_parser_enabled = true # If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for # load balancing of read queries. Otherwise, the primary will only be used for write -# queries. The primary can always be explicitely selected with our custom protocol. +# queries. The primary can always be explicitly selected with our custom protocol. primary_reads_enabled = true # So what if you wanted to implement a different hashing function, diff --git a/pgcat.toml b/pgcat.toml index 183391977..7db6abe35 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -57,7 +57,7 @@ tcp_keepalives_count = 5 # Number of seconds between keepalive packets. tcp_keepalives_interval = 5 -# Path to TLS Certficate file to use for TLS connections +# Path to TLS Certificate file to use for TLS connections # tls_certificate = "server.cert" # Path to TLS private key file to use for TLS connections # tls_private_key = "server.key" @@ -123,7 +123,7 @@ idle_timeout = 40000 connect_timeout = 3000 # User configs are structured as pool..users. -# This secion holds the credentials for users that may connect to this cluster +# This section holds the credentials for users that may connect to this cluster [pools.sharded_db.users.0] # Postgresql username username = "sharding_user" diff --git a/src/config.rs b/src/config.rs index 13528af22..4e0ff8ae2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -273,7 +273,7 @@ impl General { // These keepalive defaults should detect a dead connection within 30 seconds. // Tokio defaults to disabling keepalives which keeps dead connections around indefinitely. - // This can lead to permenant server pool exhaustion + // This can lead to permanent server pool exhaustion pub fn default_tcp_keepalives_idle() -> u64 { 5 // 5 seconds } @@ -422,7 +422,7 @@ pub struct Pool { pub shards: BTreeMap, pub users: BTreeMap, - // Note, don't put simple fields below these configs. There's a compatability issue with TOML that makes it + // Note, don't put simple fields below these configs. There's a compatibility issue with TOML that makes it // incompatible to have simple fields in TOML after complex objects. See // https://users.rust-lang.org/t/why-toml-to-string-get-error-valueaftertable/85903 } diff --git a/src/messages.rs b/src/messages.rs index 61c36c6d7..ba4818cec 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -404,7 +404,7 @@ pub fn row_description(columns: &Vec<(&str, DataType)>) -> BytesMut { let mut res = BytesMut::new(); let mut row_desc = BytesMut::new(); - // how many colums we are storing + // how many columns we are storing row_desc.put_i16(columns.len() as i16); for (name, data_type) in columns { diff --git a/src/multi_logger.rs b/src/multi_logger.rs index 901db02e5..2a61794e4 100644 --- a/src/multi_logger.rs +++ b/src/multi_logger.rs @@ -17,7 +17,7 @@ use log::{Level, Log, Metadata, Record, SetLoggerError}; // // So to summarize, if no `STDOUT_LOG` env var is present, the logger is the default logger. If `STDOUT_LOG` is set, everything // but errors, that matches the log level set in the `STDOUT_LOG` env var is sent to stdout. You can have also some esoteric configuration -// where you set `RUST_LOG=debug` and `STDOUT_LOG=info`, in here, erros will go to stderr, warns and infos to stdout and debugs to stderr. +// where you set `RUST_LOG=debug` and `STDOUT_LOG=info`, in here, errors will go to stderr, warns and infos to stdout and debugs to stderr. // pub struct MultiLogger { stderr_logger: env_logger::Logger, diff --git a/src/query_router.rs b/src/query_router.rs index 0ea907b56..5b2ba0c48 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1,4 +1,4 @@ -/// Route queries automatically based on explicitely requested +/// Route queries automatically based on explicitly requested /// or implied query characteristics. use bytes::{Buf, BytesMut}; use log::{debug, error}; diff --git a/src/stats.rs b/src/stats.rs index 5b7895b4e..b95a143a2 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -66,7 +66,7 @@ impl Reporter { CLIENT_STATS.write().insert(client_id, stats); } - /// Reports a client is disconecting from the pooler. + /// Reports a client is disconnecting from the pooler. fn client_disconnecting(&self, client_id: i32) { CLIENT_STATS.write().remove(&client_id); } @@ -76,7 +76,7 @@ impl Reporter { fn server_register(&self, server_id: i32, stats: Arc) { SERVER_STATS.write().insert(server_id, stats); } - /// Reports a server connection is disconecting from the pooler. + /// Reports a server connection is disconnecting from the pooler. fn server_disconnecting(&self, server_id: i32) { SERVER_STATS.write().remove(&server_id); } diff --git a/src/stats/client.rs b/src/stats/client.rs index 892350687..4cbcab2f7 100644 --- a/src/stats/client.rs +++ b/src/stats/client.rs @@ -92,7 +92,7 @@ impl ClientStats { } } - /// Reports a client is disconecting from the pooler and + /// Reports a client is disconnecting from the pooler and /// update metrics on the corresponding pool. pub fn disconnect(&self) { self.reporter.client_disconnecting(self.client_id); @@ -140,7 +140,7 @@ impl ClientStats { self.error_count.fetch_add(1, Ordering::Relaxed); } - /// Reportes the time spent by a client waiting to get a healthy connection from the pool + /// Reporters the time spent by a client waiting to get a healthy connection from the pool pub fn checkout_time(&self, microseconds: u64) { self.total_wait_time .fetch_add(microseconds, Ordering::Relaxed); diff --git a/src/stats/server.rs b/src/stats/server.rs index 08968a12b..d25f3b4eb 100644 --- a/src/stats/server.rs +++ b/src/stats/server.rs @@ -102,7 +102,7 @@ impl ServerStats { self.state.store(ServerState::Idle, Ordering::Relaxed); } - /// Reports a server connection is disconecting from the pooler. + /// Reports a server connection is disconnecting from the pooler. /// Also updates metrics on the pool regarding server usage. pub fn disconnect(&self) { self.reporter.server_disconnecting(self.server_id); diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index b611e8cb3..fceb95bf7 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -37,9 +37,9 @@ describe "SHOW POOLS" do context "bad credentials" do it "does not change any stats" do - bad_passsword_url = URI(pgcat_conn_str) - bad_passsword_url.password = "wrong" - expect { PG::connect("#{bad_passsword_url.to_s}?application_name=bad_password") }.to raise_error(PG::ConnectionBad) + bad_password_url = URI(pgcat_conn_str) + bad_password_url.password = "wrong" + expect { PG::connect("#{bad_password_url.to_s}?application_name=bad_password") }.to raise_error(PG::ConnectionBad) sleep(1) admin_conn = PG::connect(processes.pgcat.admin_connection_string) diff --git a/tests/ruby/load_balancing_spec.rb b/tests/ruby/load_balancing_spec.rb index cd647406c..172e6069b 100644 --- a/tests/ruby/load_balancing_spec.rb +++ b/tests/ruby/load_balancing_spec.rb @@ -65,7 +65,7 @@ processes.pgcat.shutdown end - context "under homogenous load" do + context "under homogeneous load" do it "balances query volume between all instances" do conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) diff --git a/tests/ruby/sharding_spec.rb b/tests/ruby/sharding_spec.rb index 4c4053de8..123c10dce 100644 --- a/tests/ruby/sharding_spec.rb +++ b/tests/ruby/sharding_spec.rb @@ -27,7 +27,7 @@ processes.pgcat.shutdown end - describe "automatic routing of extended procotol" do + describe "automatic routing of extended protocol" do it "can do it" do conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) conn.exec("SET SERVER ROLE TO 'auto'") From 98616cd62d5db2b6fd6fb0a0f08d9ffe5512b0de Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 11 Apr 2023 10:32:48 -0700 Subject: [PATCH 378/551] Load config on client connect (#401) --- src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 39c67c2ef..acd19352b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -287,7 +287,7 @@ fn main() -> Result<(), Box> { let drain_tx = drain_tx.clone(); let client_server_map = client_server_map.clone(); - let tls_certificate = config.general.tls_certificate.clone(); + let tls_certificate = get_config().general.tls_certificate.clone(); tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); @@ -298,7 +298,7 @@ fn main() -> Result<(), Box> { shutdown_rx, drain_tx, admin_only, - tls_certificate.clone(), + tls_certificate, config.general.log_client_connections, ) .await @@ -306,7 +306,7 @@ fn main() -> Result<(), Box> { Ok(()) => { let duration = chrono::offset::Utc::now().naive_utc() - start; - if config.general.log_client_disconnections { + if get_config().general.log_client_disconnections { info!( "Client {:?} disconnected, session duration: {}", addr, From 6220da0f127326772e50b8daf935bc3169a8c0e1 Mon Sep 17 00:00:00 2001 From: Cluas Date: Thu, 13 Apr 2023 00:29:43 +0800 Subject: [PATCH 379/551] feat: set keepalive for pgcat server itself (#402) * feat: set keepalive for pgcat server self * docs: note also set for client --- CONFIG.md | 4 ++-- src/main.rs | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 667daf12c..613da7522 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -1,4 +1,4 @@ -# PgCat Configurations +# PgCat Configurations ## `general` Section ### host @@ -127,7 +127,7 @@ path: general.tcp_keepalives_idle default: 5 ``` -Number of seconds of connection idleness to wait before sending a keepalive packet to the server. +Number of seconds of connection idleness to wait before sending a keepalive packet to the server and client. ### tcp_keepalives_count ``` diff --git a/src/main.rs b/src/main.rs index acd19352b..b3265ed87 100644 --- a/src/main.rs +++ b/src/main.rs @@ -79,6 +79,7 @@ mod stats; mod tls; use crate::config::{get_config, reload_config, VERSION}; +use crate::messages::configure_socket; use crate::pool::{ClientServerMap, ConnectionPool}; use crate::prometheus::start_metric_server; use crate::stats::{Collector, Reporter, REPORTER}; @@ -289,6 +290,8 @@ fn main() -> Result<(), Box> { let tls_certificate = get_config().general.tls_certificate.clone(); + configure_socket(&socket); + tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); From 8fdb6e598665f80a9bc01a19545b4177b6895676 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Apr 2023 10:23:52 -0700 Subject: [PATCH 380/551] chore(deps): bump hyper from 0.14.25 to 0.14.26 (#406) Bumps [hyper](https://github.com/hyperium/hyper) from 0.14.25 to 0.14.26. - [Release notes](https://github.com/hyperium/hyper/releases) - [Changelog](https://github.com/hyperium/hyper/blob/v0.14.26/CHANGELOG.md) - [Commits](https://github.com/hyperium/hyper/compare/v0.14.25...v0.14.26) --- updated-dependencies: - dependency-name: hyper dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d6c56f05..09916dae4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,9 +387,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" dependencies = [ "bytes", "fnv", @@ -476,9 +476,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ "bytes", "futures-channel", From 4bb576a12d1736339c27773ad310469a5ffb99e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Apr 2023 10:24:44 -0700 Subject: [PATCH 381/551] chore(deps): bump sqlparser from 0.32.0 to 0.33.0 (#399) Bumps [sqlparser](https://github.com/sqlparser-rs/sqlparser-rs) from 0.32.0 to 0.33.0. - [Release notes](https://github.com/sqlparser-rs/sqlparser-rs/releases) - [Changelog](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/sqlparser-rs/sqlparser-rs/compare/v0.32.0...v0.33.0) --- updated-dependencies: - dependency-name: sqlparser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09916dae4..cc5f40949 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1108,9 +1108,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0366f270dbabb5cc2e4c88427dc4c08bba144f81e32fbd459a013f26a4d16aa0" +checksum = "355dc4d4b6207ca8a3434fc587db0a8016130a574dbcdbfb93d7f7b5bc5b211a" dependencies = [ "log", ] diff --git a/Cargo.toml b/Cargo.toml index 4e33c645f..a5573518e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = "0.32.0" +sqlparser = "0.33.0" log = "0.4" arc-swap = "1" env_logger = "0.10" From eab9b082f687d33be800bffe9773ec300be90ad2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Apr 2023 10:24:52 -0700 Subject: [PATCH 382/551] chore(deps): bump serde_derive from 1.0.159 to 1.0.160 (#403) Bumps [serde_derive](https://github.com/serde-rs/serde) from 1.0.159 to 1.0.160. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.159...v1.0.160) --- updated-dependencies: - dependency-name: serde_derive dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc5f40949..f47951e28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,9 +1020,9 @@ checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" [[package]] name = "serde_derive" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ "proc-macro2", "quote", From 4a5d3312506fbec661b187079e6687361cbf6620 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Apr 2023 10:25:00 -0700 Subject: [PATCH 383/551] chore(deps): bump serde from 1.0.159 to 1.0.160 (#404) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.159 to 1.0.160. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.159...v1.0.160) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f47951e28..e0027296e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1014,9 +1014,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" [[package]] name = "serde_derive" From 1416bffb9d5782aba577be72622a3ee7c878cbc1 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 18 Apr 2023 09:57:17 -0700 Subject: [PATCH 384/551] Separate server and client passwords optionally (#407) * Separate server and user passwords * config --- CONFIG.md | 92 ++++++++++++-------------------------- pgcat.toml | 16 ++++++- src/auth_passthrough.rs | 2 + src/client.rs | 5 +++ src/config.rs | 4 ++ src/server.rs | 30 +++++++++---- utilities/requirements.txt | 1 + 7 files changed, 76 insertions(+), 74 deletions(-) create mode 100644 utilities/requirements.txt diff --git a/CONFIG.md b/CONFIG.md index 613da7522..fbee178fe 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -1,4 +1,4 @@ -# PgCat Configurations +# PgCat Configurations ## `general` Section ### host @@ -108,7 +108,7 @@ If we should log client disconnections ### autoreload ``` path: general.autoreload -default: false +default: 15000 ``` When set to true, PgCat reloads configs if it detects a change in the config file. @@ -127,7 +127,7 @@ path: general.tcp_keepalives_idle default: 5 ``` -Number of seconds of connection idleness to wait before sending a keepalive packet to the server and client. +Number of seconds of connection idleness to wait before sending a keepalive packet to the server. ### tcp_keepalives_count ``` @@ -175,41 +175,11 @@ Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DAT ### admin_password ``` path: general.admin_password -default: +default: "admin_pass" ``` Password to access the virtual administrative database -### auth_query (experimental) -``` -path: general.auth_query -default: -``` - -Query to be sent to servers to obtain the hash used for md5 authentication. The connection will be -established using the database configured in the pool. This parameter is inherited by every pool -and can be redefined in pool configuration. - -### auth_query_user (experimental) -``` -path: general.auth_query_user -default: -``` - -User to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query -specified in `auth_query_user`. The connection will be established using the database configured in the pool. -This parameter is inherited by every pool and can be redefined in pool configuration. - -### auth_query_password (experimental) -``` -path: general.auth_query_password -default: -``` - -Password to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query -specified in `auth_query_user`. The connection will be established using the database configured in the pool. -This parameter is inherited by every pool and can be redefined in pool configuration. - ## `pools.` Section ### pool_mode @@ -243,7 +213,7 @@ If the client doesn't specify, PgCat routes traffic to this role by default. `replica` round-robin between replicas only without touching the primary, `primary` all queries go to the primary unless otherwise specified. -### query_parser_enabled (experimental) +### query_parser_enabled ``` path: pools..query_parser_enabled default: true @@ -264,7 +234,7 @@ If the query parser is enabled and this setting is enabled, the primary will be load balancing of read queries. Otherwise, the primary will only be used for write queries. The primary can always be explicitly selected with our custom protocol. -### sharding_key_regex (experimental) +### sharding_key_regex ``` path: pools..sharding_key_regex default: @@ -286,7 +256,7 @@ Current options: `pg_bigint_hash`: PARTITION BY HASH (Postgres hashing function) `sha1`: A hashing function based on SHA1 -### automatic_sharding_key (experimental) +### automatic_sharding_key ``` path: pools..automatic_sharding_key default: @@ -311,47 +281,43 @@ default: 3000 Connect timeout can be overwritten in the pool -### auth_query (experimental) -``` -path: general.auth_query -default: -``` - -Auth query can be overwritten in the pool +## `pools..users.` Section -### auth_query_user (experimental) +### username ``` -path: general.auth_query_user -default: +path: pools..users..username +default: "sharding_user" ``` -Auth query user can be overwritten in the pool +PostgreSQL username used to authenticate the user and connect to the server +if `server_username` is not set. -### auth_query_password (experimental) +### password ``` -path: general.auth_query_password -default: +path: pools..users..password +default: "sharding_user" ``` -Auth query password can be overwritten in the pool +PostgreSQL password used to authenticate the user and connect to the server +if `server_password` is not set. -## `pools..users.` Section - -### username +### server_username ``` -path: pools..users..username -default: "sharding_user" +path: pools..users..server_username +default: +example: "another_user" ``` -Postgresql username +PostgreSQL username used to connect to the server. -### password +### server_password ``` -path: pools..users..password -default: "sharding_user" +path: pools..users..server_password +default: +example: "another_password" ``` -Postgresql password +PostgreSQL password used to connect to the server. ### pool_size ``` @@ -382,7 +348,7 @@ default: [["127.0.0.1", 5432, "primary"], ["localhost", 5432, "replica"]] Array of servers in the shard, each server entry is an array of `[host, port, role]` -### mirrors (experimental) +### mirrors ``` path: pools..shards..mirrors default: diff --git a/pgcat.toml b/pgcat.toml index 7db6abe35..c9faef8fe 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -125,10 +125,22 @@ connect_timeout = 3000 # User configs are structured as pool..users. # This section holds the credentials for users that may connect to this cluster [pools.sharded_db.users.0] -# Postgresql username +# PostgreSQL username used to authenticate the user and connect to the server +# if `server_username` is not set. username = "sharding_user" -# Postgresql password + +# PostgreSQL password used to authenticate the user and connect to the server +# if `server_password` is not set. password = "sharding_user" + +pool_mode = "session" + +# PostgreSQL username used to connect to the server. +# server_username = "another_user" + +# PostgreSQL password used to connect to the server. +# server_password = "another_password" + # Maximum number of server connections that can be established for this user # The maximum number of connection from a single Pgcat process to any database in the cluster # is the sum of pool_size across all users. diff --git a/src/auth_passthrough.rs b/src/auth_passthrough.rs index f313dead0..4e9362623 100644 --- a/src/auth_passthrough.rs +++ b/src/auth_passthrough.rs @@ -72,6 +72,8 @@ impl AuthPassthrough { let auth_user = crate::config::User { username: self.user.clone(), password: Some(self.password.clone()), + server_username: None, + server_password: None, pool_size: 1, statement_timeout: 0, pool_mode: None, diff --git a/src/client.rs b/src/client.rs index 4114e4bb2..5098ec6f5 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1128,6 +1128,11 @@ where self.buffer.put(&message[..]); } + // Close the prepared statement. + 'C' => { + self.buffer.put(&message[..]); + } + // Execute // Execute a prepared statement prepared in `P` and bound in `B`. 'E' => { diff --git a/src/config.rs b/src/config.rs index 4e0ff8ae2..00c0f0680 100644 --- a/src/config.rs +++ b/src/config.rs @@ -178,6 +178,8 @@ impl Address { pub struct User { pub username: String, pub password: Option, + pub server_username: Option, + pub server_password: Option, pub pool_size: u32, pub pool_mode: Option, #[serde(default)] // 0 @@ -189,6 +191,8 @@ impl Default for User { User { username: String::from("postgres"), password: None, + server_username: None, + server_password: None, pool_size: 15, statement_timeout: 0, pool_mode: None, diff --git a/src/server.rs b/src/server.rs index 14862bd05..84bed6ccf 100644 --- a/src/server.rs +++ b/src/server.rs @@ -103,19 +103,32 @@ impl Server { trace!("Sending StartupMessage"); // StartupMessage - startup(&mut stream, &user.username, database).await?; + let username = match user.server_username { + Some(ref server_username) => server_username, + None => &user.username, + }; + + let password = match user.server_password { + Some(ref server_password) => Some(server_password), + None => match user.password { + Some(ref password) => Some(password), + None => None, + }, + }; + + startup(&mut stream, username, database).await?; let mut server_info = BytesMut::new(); let mut process_id: i32 = 0; let mut secret_key: i32 = 0; - let server_identifier = ServerIdentifier::new(&user.username, &database); + let server_identifier = ServerIdentifier::new(username, &database); // We'll be handling multiple packets, but they will all be structured the same. // We'll loop here until this exchange is complete. - let mut scram: Option = None; - if let Some(password) = &user.password.clone() { - scram = Some(ScramSha256::new(password)); - } + let mut scram: Option = match password { + Some(password) => Some(ScramSha256::new(password)), + None => None, + }; loop { let code = match stream.read_u8().await { @@ -172,11 +185,10 @@ impl Server { } }; - match &user.password { + match password { // Using plaintext password Some(password) => { - md5_password(&mut stream, &user.username, password, &salt[..]) - .await? + md5_password(&mut stream, username, password, &salt[..]).await? } // Using auth passthrough, in this case we should already have a diff --git a/utilities/requirements.txt b/utilities/requirements.txt new file mode 100644 index 000000000..aab392a3a --- /dev/null +++ b/utilities/requirements.txt @@ -0,0 +1 @@ +tomli From acc0438082f37008cd663215bcc2d932b53f411d Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 19 Apr 2023 07:45:26 -0700 Subject: [PATCH 385/551] Fix CONFIG (#408) Fix readme --- CONFIG.md | 33 +++++++++++++++++++++++++++++++++ pgcat.toml | 15 +++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/CONFIG.md b/CONFIG.md index fbee178fe..72d5683a1 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -256,6 +256,39 @@ Current options: `pg_bigint_hash`: PARTITION BY HASH (Postgres hashing function) `sha1`: A hashing function based on SHA1 +### auth_query +``` +path: pools..auth_query +default: +example: "SELECT $1" +``` + +Query to be sent to servers to obtain the hash used for md5 authentication. The connection will be +established using the database configured in the pool. This parameter is inherited by every pool +and can be redefined in pool configuration. + +### auth_query_user +``` +path: pools..auth_query_user +default: +example: "sharding_user" +``` + +User to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +specified in `auth_query_user`. The connection will be established using the database configured in the pool. +This parameter is inherited by every pool and can be redefined in pool configuration. + +### auth_query_password +``` +path: pools..auth_query_password +default: +example: "sharding_user" +``` + +Password to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +specified in `auth_query_user`. The connection will be established using the database configured in the pool. +This parameter is inherited by every pool and can be redefined in pool configuration. + ### automatic_sharding_key ``` path: pools..automatic_sharding_key diff --git a/pgcat.toml b/pgcat.toml index c9faef8fe..6cb9a2991 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -113,6 +113,21 @@ primary_reads_enabled = true # `sha1`: A hashing function based on SHA1 sharding_function = "pg_bigint_hash" +# Query to be sent to servers to obtain the hash used for md5 authentication. The connection will be +# established using the database configured in the pool. This parameter is inherited by every pool +# and can be redefined in pool configuration. +# auth_query = "SELECT $1" + +# User to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +# specified in `auth_query_user`. The connection will be established using the database configured in the pool. +# This parameter is inherited by every pool and can be redefined in pool configuration. +# auth_query_user = "sharding_user" + +# Password to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +# specified in `auth_query_user`. The connection will be established using the database configured in the pool. +# This parameter is inherited by every pool and can be redefined in pool configuration. +# auth_query_password = "sharding_user" + # Automatically parse this from queries and route queries to the right shard! # automatic_sharding_key = "data.id" From 929666f96a2ce71328b576801715cb21b9062477 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 20 Apr 2023 07:53:55 -0700 Subject: [PATCH 386/551] README updates (#409) * Better table * add image * promote auth passthrough to stable * fmt --- README.md | 46 ++++++++++++++++++++++++++++++++++------- images/one_signal.webp | Bin 0 -> 16278 bytes 2 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 images/one_signal.webp diff --git a/README.md b/README.md index 56a860573..4dd752460 100644 --- a/README.md +++ b/README.md @@ -21,21 +21,53 @@ PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load bal | Client TLS | **Stable** | Clients can connect to the pooler using TLS/SSL. | | Client/Server authentication | **Stable** | Clients can connect using MD5 authentication, supported by `libpq` and all Postgres client drivers. PgCat can connect to Postgres using MD5 and SCRAM-SHA-256. | | Live configuration reloading | **Stable** | Identical to PgBouncer; all settings can be reloaded dynamically (except `host` and `port`). | +| Auth passthrough | **Stable** | MD5 password authentication can be configured to use an `auth_query` so no cleartext passwords are needed in the config file.| | Sharding using extended SQL syntax | **Experimental** | Clients can dynamically configure the pooler to route queries to specific shards. | | Sharding using comments parsing/Regex | **Experimental** | Clients can include shard information (sharding key, shard ID) in the query comments. | | Automatic sharding | **Experimental** | PgCat can parse queries, detect sharding keys automatically, and route queries to the correct shard. | | Mirroring | **Experimental** | Mirror queries between multiple databases in order to test servers with realistic production traffic. | -| Auth passthrough | **Experimental** | MD5 password authentication can be configured to use an `auth_query` so no cleartext passwords are needed in the config file. | ## Status -PgCat is stable and used in production to serve hundreds of thousands of queries per second. Some features remain experimental and are being actively developed. They are optional and can be enabled through configuration. - -| | | -|-|-| -||| -| [Instacart](https://tech.instacart.com/adopting-pgcat-a-nextgen-postgres-proxy-3cf284e68c2f) | [PostgresML](https://postgresml.org/blog/scaling-postgresml-to-one-million-requests-per-second) | +PgCat is stable and used in production to serve hundreds of thousands of queries per second. + + + + + + + + + + + + +
+ + + + + + + + + + + +
+ + Instacart + + + + PostgresML + + + OneSignal +
+ +Some features remain experimental and are being actively developed. They are optional and can be enabled through configuration. ## Deployment diff --git a/images/one_signal.webp b/images/one_signal.webp new file mode 100644 index 0000000000000000000000000000000000000000..79f3d1293df27aff07988ec9fed98941a10f7c19 GIT binary patch literal 16278 zcmeIZQ?PJN(0H zotQ#=b6wEqhk_NdZ}`U?9Noy}-{9}2d$Q%wjGx+%*t~b#Pw^-8CmucD!DqVH>Kmqy z{Ex>U*-g9^vJAeBT2){9+#TO#PXJ%z-}05$Gmg*v^X04KZ{HuX64r{>gx(on=NGO$ z+tc3m56|Vwjom2E@eSVxxXi$qgczWl;ncT$@b|`u!clk9&*mSm}Yl5eu zj8VJ*8?3JUKc6HJau6cU**p<>XuaZG>->m4Hj0S0p9gwAWu|-Q%l?Ms)ia887a?_x0U{O`HxXHB6e9|OrFJ1To5P*>Nt>G_nsiNeaePdZ?< zyXx3cVVKqd1r7~pCo0wr1p@wiAskeW-Zcy%`FgxIrt8s;5d0s%a<`?^B+hjQ;3qP| zUV@i#Qd-o;Z1saw3Kwrjge2{;Qq%cQ8DLs{oX0Q?7+QMx@~ik){zG%3q!FXE6KDrb zDm<)7JN<$BM;zY7&)g8PHFf&{?IJnKj(pPATDV2dH0hn@@VKYHsO?j)S?c)R=Ke!f zAWD_9Ef}GA&y2oSLPpg-vg$B1KQ0Iy(f*4SkmK>09Ffo~|5Bl2g)AGsU(`Q3{k4!h zqRAJh33TfGah&OJeH>i*08qxDuB@KO`+u1CWwoeH4G4?r9p^fg>6iS73<>A3(+#n5_=md)1Ui=P z_K5gC#1lkr^wDu)0G{#*@{YV=5MMs;~v?d-vH_1*fFaL7Jg7 zZyuNAD(wBREhRyj(Hg{d(jwC#{mYBX6Qo3L^5E~j7xSj!c0Id6&_F{v!i$_1a zD(2NtLy#nM=x85LMfvo-I`~WM!F9raPl3W-{!Xh4@~j(w2;(3OZFW9R&at&V|CS^> zrex_{rf52NgILUxY@+DZ(f-;$TeoN#HQ)hGr0~`(EkE;HS_q45{mt5@un*8nNoz1H zPk0$wb3WO641mn_)CW0{kgKS~;7h!lCw6 zLXAU^L{%a~AgEi}w#CXx+i{N`M48258s5~$d?Z+|s>LBIN9`~4P9G{Qc&yGqn37ek zGA8BFu+b>%qpMv3p*<@$s3x9W`ok)8AUI|?23CzrBkl#Yl{Iv7uRG7ax9SndH*CdJ zvrW7lz6U6ma!0@~#u=PAh)6^kLJCGExu?GWsN_Vnw5T4_CyrDRCGIcp;g#|AUQKY- zivVmJJM{`F&x%(dutxf2wy}l?)|$kOOn=z>qV}J*w&HG@8rOaL`ZT4+rtM2}%>r6W zy)e1QMo%C2(r$0MU_7KvQ>ZI#8HG#z8Z$zTOV4=?a_C{F4`v3>X3o4z-1|l6BYyhs zy9{C`S*3_Z#3?G4lG8Rq{~H=cD{Cpj1X{d|Dd-67QJykd99iiSwxm0q%S-pe3>!IN zpi7+H3-pHPjPM-K&f=DVayi;S2NJHHXW#1|7~cm0`dqjtCVZ;|c*N1&QNv+j0T_6L zix|KWRF(MkXFNy>UDGkvb>Oz)c*Os)`(-2qU2;gZtx)>A_or|Q6Fn{E4!NmGK*Ls` z|IpqA=HoXW71phw(>EkL8K|*dn-e3aF2=_~H5!mX5(U*tNJ{AKGj<)lAIr!j|J?6C z1XI)nn35V|YlL5@&JZ5Pzp`)nK^X@0=jMkPzwvLwB^HprFf%xl`%cv4{au;L5Y*dG z;;W!{p@I%!0vai5{ikioI36-_xjxws0`0`~Sb!LeCyVwbHJ_x%T5ZFEX0)LL%VHVS zH8XFAtow&leH+ulCiKd{BKKaQ*|4u(cK2gVf6iJc8B(|^(Oew-WxCJ5OJ4)M7^jV= zyxrd*IUf~H+GJp_B@S)G|bdJTo7JHh++F=)NR#<^ja|;1*M^-Fa=1TYd}TX#Ns* z^F)Fy5nG+9h={05|D``yS6-%0F8VpC8+4Kk-vPRx_g+VAUiFQ7Wt#T#dC#K%w_0Qj zEF*my(gh0cy-Y)-9p6ulV=0kT1sQBUKK$F7b2S%jP&;wb6>kaMD?R7-PjRuXwLJu; zN`e(Q6jR4?zmy_fCp?(96tjr!1b9hwWCQ*Fq55f&(+k`9@^jrBh=&%UsC zM=kJXhx3Mmm@62C9m%G434Uk)RO5|_h$528LFp-|w5R8VRk-Ju}tWDU4t1tS{7VPARO<8ujKC`396*LM0|xLiR2 z+1R?Yz1js~jXR_MoScH*bG73Nc!Olf_PPi4OoK$9?qT!MSUL-8XC=C^=U-~yt;-($ z0arQ-Rtrs!oFj2n7C2>1$XG$R#cb99!3OPZuY}1xrs=E2%(WqG$dD&;<|S|&jj3a+ zzJ(;5CMpn+@$FY|x#gV?Z6MRo-(BJySl--hFz_l572zYY9*u=TJv1HnI_=kTLF(KN ze)i1ts@08YjmR#Gq5l^~$e-Q72jUH`m%h()PO-_RJ&7KUY&nR59b$W@(pCbuD8&b8N}%>8qahGD!eOoAU0S@V z)gU8^dCdeZl9b}skr@P*0?Dxox=DO4F16wGJT`t$x^OZ?@P`O6Uw;2=92gVkOU$%7 zZz+wt1SuJD%Eo3QM)&1V!>eV?H|zr+hu&*0CPd!agqgq-Wjy&D+y6Uk_UziNNHr1% z=m>eKJktDzrS4G6!sTfF%de?P*hJ_0Jc^IaD3TBhhcMHG5w4AL`k15=QjU|E^*};< zD)86swiCjXT!IVLcE-V7fuH?puL2WJs`i$+F!caP;)@DWt(I{zUUzbwMO67c+wktl zLC*A?(&!VPq3syzQg%e2cCE75z=*_n!LkSuLOw~Px^OMd0C z^s-rm;8NAeDC}P>z+7Z60I*dJef*o3;(!Q&URw^a`x(jsD&AdD-z&ZFPBs9|-8H-y z-ypQ9=tg%SoTGQi(BUTq|Lu%jEKXP`OER?ng)&|PF5z8Lt=@n?*x_^4yQrKOVu%XV zM;zl1Sh+p4aTO;-vAJe+q0$aNi&1fRx{~FZINX{QlLEU6=(t7H>sy%CgkOe&t1Iw? zIM0mZ8pHy4R@0{j4<033~3x#vjff}@s)HOiNXzF1}2)e= z<45H&2&UbB+It@>EH+@oQe3W#Ie?qRdVDED|6%CveJerZrvlGz$sF@U)|mbyWG4YR zlBhTSpd34|#!)E{=xPrFI22&Nf&i2krsssU-)ujJlT z2YdcazPFq8niy$%ULI31SaH3sU07RT$kN|0T&SGd8774sI)NkS(QsIz^29Q(MN{JX2ra*ms% z@S_p{Alni9{VxstA;lWSkVf;GVRNf`3SLWiY_a=K5Sa)vxBqk8u0W)!<5&!NxmBxW%^hWZ4qVWC5kok1XzjTwh5$h zint8K0aH;te;h0UTQUb*ISb3HbfkY#cOvas?(ph)IrJ8Se{KidMiO;%{k;U*ydj6! z#ZAkDvzSNt^2}H|TgQgx?&-J_dQ?JCoe;pe9ia;!$?D4LPFE!0&0vVZY`fv+LLLy2 z>o0w1Gqrr8iL~_P{1>~_P3csV{Jc5a>5A-&q)-D10KE>YFxtTw;6epCb_!CY3j=_Z zoo|9=jH5|@3|uOA5yv)7IH7pl=0*ReW2B{y)mxe1sm!TyPpimeDjz4wzSn7K8l*WB z?qwlv-o0yys|Tc)ZDSufS*zAB%iG1)=EP&AeBS-S3O#3bX)D8aqE2}sgV4%B29;hd zfDxK54*QlmH=ekhY5Z>_d_g9Zm+5)X3834y{i+Rwa93*%&Q>klRV<6SzBV6OnSB4+ zfG}m4R~RFszYjX_l32o|mbu?M;U^+Vwu3a49-awT)LItN0TYn1FNQ>7WpSR|OZCa| zrW%6lzUQfPMB)?M{$o^8GJms1!V)Ac5l^Bx!7HjKygA>Q>m6f0nLE8kn7_ScUs4gw zK(4{C5EO#3Owa*p4a!~9!oyPg=OARIRP#b%BnQ{xVQ|69YN6Slp*gfO}6qwCiQ zo8{u@-%7SXQ=*^-Bcl`nfX+q5S4q+(JvlAMI-%APt+RmPV+j* z*hw2fCx1>11tBcw`;=NPgc-{O@5Ycfi@u$zAX3%vtfG#iUSwn-N8@$?|?Nk&2>(X8O*F=i)3MTW1~x zo}@62#a-zP>9tF5P+fz|A~tGKUt1`%h$>z&;Cm-}7Wc4Sb}iDrW%shZ7CMmKO>Vs8 zC7oABhZW#k+uZ0auweI|QzVu9V)tDX+?}?d4Lqc3FHN@`Ns^N=jw3FOnl#WJMV5L@ zklkz<5HB#4?3NTc5qWm1O0`x=4kP7rO=N?Xk)2g^!pINA7JkFHr=VIWS9QJ?m|7Pd zp^_&tW%QiPhtwo;8O6!*`y^P7$6DmEuyF@)KW1*-U0D2bdPH?j3j^QX2K=&}g8qZg zPxX__r}2w~h=iBwaqD1nBgu9oau6Xy=#WXTsh`rfiu6{^0Yx6N=-zspciE;1Tk~V% z7JIVmL@UPtT2*xVTVF5#~Wk>ycf?g3< z^kmTFK6NeoAF0M7QWx2XzQf$fCA?@&q|pg@+d7 zYWu6~HOEYNtpO*@RK?edeWkGRJtGP9=#iek!&N3>HE_^$C@R`l-X=yetWd8OZL z6cs4%pih$4fVUn{@u7GtiKE|@mxBW5tQU}8h&%)tb3TN(__~7dl zgueswvDBESWR(HzD$|v*3XR}2LrNNuZz3n$_6$5`84qQL9yvQFc1x!$skUmF3TMko z$Gse z3a;eJcL68S=8n+ICKwkaI;uH){BEe{DOW~TR!fC_HCs68`B#Hz<#@Ld;@X`xftbh4 zNdnLGc?J6=`r~QGfr9=aM6~6hf$3Q_*)~ztMe5UCMORm(!3df-QMVg0J;VKNGfH#P zH!doK^eL+~W`^Zg%xIFkn~;)^t~^-5c#_P+0{;72FSv0|qJq_U_MMj^_>BNTjv5=N zvwgDjO6fJ8BQdiixG%)ADLNuNrY^=~ZW4ABeL!)ZgIMd3&izOyPZRr>Od_vLI7!Xk z#2R&kahVB)o1c~p`qdDarmW9O`v>&MchzHnE!0U0w8YnB^OM|KDcEE+HlBTIb^n`JBQzxwB6vHZdp9^<+jBRW< zT7Cd^0%cxuD2vl)~T??vC~jB)!U zSAd`-(s7qh9nRrN7ZgnO3k}a(aanLqm}om{)B4f;MD<=nbIw1ft1tagiq~!(C%U z`$E~&Wxt*20MsUbFgR(fiTEm(4mvqgZ335@73xg|>5D{!s4+Oc(b3@N6N_?rJs+>& zvu|s{u=fa8@2?dEn*j{NLN=H$BPwTZJ%cFO`#ofPTWK%0~p zN7}8n=FcDizGY04ZM8bosnP&M)_jEE9z7p)4r;A}ZjbA-4w_WqGej})@59xMf+M_J z-QYFE_dBkYOpWMbjxa@4XPe!;H)}R^^J|@?HTwKYekYy^Q0?N*sph&@9leRG)7o|W zxs2PYga&=6(<-pqg0?4Mj#H^}?!ZE5ou$ZoWfWcOzLf+(NEf*3;R-FcVLzsE&|ibD zC7nPTNNvrLr}@O4-q*mAw!K@93_edUw>fg8mjU}#!k( z{0?UtKLzr_wy&=A4cHfbJ_ZYcF&wqc`Ci`a`tUE>;o!sO<=xJe z=%m9LzaEtEb6#N3D*>N1$H^28Xc9Ngo+W-xS(?!KuWl1FbUOI?0PV8#H|A6}r8uCj zJcX4(#xfxM;x0T{CCu)1HN^L^=6HE7PE4U5;~1VwIK)AWF#+e062*W=AwY;yZa`;i z4bEJ^lW@vQ)loW{5s6&5AeEvzTQ@uxL0FU?QL}vVkaoRrC^M|HKk3-XE`fR$W%U5- z5S?!W!2t3oj2U43);h7EwJ%;+E*qKCb+R|%J)nnsPljDX>d*7QkveEaQT+Yd*0|Ms z7>2a??-glJ-l##?OTZP=nVTS67jn<3@){B!A}c7mzEKq`(44grQG1oKStp-+DoUTH z&EP{Hc6dF2fg3Kf+jOcS-Tt1frs7+`PLoUOW9g(gq!OM^4_-zUty3!YV!>{`R7rjV zVedMP4d5lx!QnSy0agVTBhWX=tbwC^5yF^6sb&(aci(}}I4Yh>jZo$-Jm7W`CJs4B zb<@QFMrFF<(EuDMrb*JKBz*Pl8W!kjDAeQ}LT{_4F=WzNs%@G$p1Z1e4%v0Cc=MVq zP7YVdC?7;T;uPvOJoU&RTdSMoV2-?uDE^iVf}|<&Kwe&6H-1`8spAJ`>J-m8^UWE# zYnDl-5LvJ*DjDJt;K)c0lD79WQeN=MVdUnz$oJAO@Qv>;zvaC>2=y1|+E9%0C< zC~SmN;WWz@KvfCKBcXkzb>e4X-u>vx5dnu0fE+f_+^(6|A$)=R-b7X+M?X!zM}^HZ zFa#-o`z-$+((YTu5Du%D+z6WhuQHI@r)9h&HfInN>#t&W2}CsW5Zh1w4pNU9x!%ID z#!m4!#m+)_#OHkm*4kW8AdcpCq?X`OrCJvo0T*D&x8VJtg%BZ~!%egfTj({aW0R?s zJQ^EEkPRM;q@AP#t}^>b>d6|yC~Z3@R*4E>Wf%2|U^_wbA`oW%so+}p74!uJyMde+ z_~pRlK-@Fl8)Fm-VukhIYWF~GDZ#3#8r7@~LhsD`tEaup=|^N-0pW5Tvq)jgj;+$|y@$fGMgqZq7uW->Qq|2ETQ#Z_ zGI)Zq!5l==EQzMNB-LZK3kuCTX&Ssa5tW+%BNie%<;ezBSNkwyC*Cxlf{r%bRi$qc z7xqc76q}U@31}NR6TzMQFs>B))tQ#Tom93ycrQ}Eb}byu-_|=jsX&L5Z9X^r0%8+e zOLfDOmSU+wn!YqG#wrTkJ|gN;GFmI%9V%SYVti}Ss4kqh$MPuD^2m~RpfG|M3Xc?k z+&rSK-HY*rPbFx`2B~GwStt(L@FH$K_a)A6H~~aZ9BI$BZeaY9HwZFT1TflXFJgSl z*KEw*qB%>jsEk_$Jg{9qf}dlXA2vJ7mZGEWBzb(j{;qnA&s4pao3)!Mqdt)%P$n!O zn25c?@>dgfKu$1VrmgftcN~A3|I|v6AHSTPj`>#07Bq~Fq*U9-Mxi~1uoi|~=^1Ks zzZTbpl7F6mX?^GJq=*Ehr6%eJ=2fXRx!^LWT;kiB1+VPAEq^7);*!O=Xi~DoxONqc z!lSZ8*dXacI=GXsTlK-*1f;@0GP|UL!(I0^S7}QZo7Laf=$kPP-s3b`f^gi{ginRm zCjt(3CVpJ>GVLGxOUy%Qk&%Vg;m4zDm8^n9u0H$-hliDA*cP$*15v`PAlcV@J4Ldp zn><;94FTsCV1d51RLL+g@(ahhJzyNBF(6bW)>(?%7KE33^xKY+K7I7yYbUc8Y2mG@ zyaWu77zcY*yhB3ec1JB?t%X+RAsd9$q!R}=3BZGL;O6n{oxK}$&u>|7u3d@=7vnj8{fv)>Vsnm5~Tp3j}_Q2aYlB0wJMu!i#8wo`JL(T8YDf{0bc08mcFQ?eh>u zI_ay=9-52!gJuB+ZM3bX*Zlk zw!Y=q3)of{6*NuyovC4-Ef6KGVq>_^!`Su>vhL-mO;3k8TmpTc>k}&{iqQ8|kW?6h z9>Ny-tC0D>NqoisE7NmubwXm zwsciI=eZHZKL5-0^@87sNm0}_T7DO`Tqu?Y-nuSEM=QAES3*s4wS(k9m2lY8;&@6I z@fZWsgryK8IPw7~IK!miMKWq0$P+BHl9$?njed?q@YX1WtU1>c(7`#SjAlfAC3L{n zAD}f;xt>yzETkOdHBRU>u*pb6PPn!5rPLXFK@3%gppT z?6>Q0-5ah%GqEafJ)xxZ1&gaYtknqIIa$@5dp4=wxX#*(95r4bOEQCHXb2biyH3b{ zEf)+eU!U_@nQ$Ug7tUU0kz^zxnkl#23Hu^u8QYF?#Jiwo{TriILh z3e`4##5v^!s&Ypj`)jFhM`8XKox}+CFcWSO_QdC2qUj-5*r=E2UT>E;;Hk6=SX%dK zl=g;vU#vycJ>{P3t8~p{?~C8~$PKGUHH}F!;1uJ9t0k7F-0h+6wZ(ei#2(@s7nlgn zYIhc|_^QS;UpEk{_-EhZQ{gfZ4&zJsGa?IO|D1v)0P^Q2g;>W1qV!%OFV@5rToo`p zjMP6^wvu5h>ZT?;paVr#;U$+MY3uPM)&3P~#HQP%RDxj!Km@cHQ8T7^hGH3j+n5tb zd_mlHtJ>i#e$Iv>1ECg-{H`b|gKHRLb#r}*vKWPEGxWE&Po-yEU*-*{bk+&QF-O(o zq>r)Ucd^BOLd`2LSyZrdM}Zm{bFV6Iz9GU_MFh?*&l+I#`bg2`K~QY;w=hoi$s`A2 zxmIKe`fhKlY1eF`Csm;#^mjB&9Rq5*gqOq=lAO}|fSL+A z7&$LSTyEcaZqXE$&XNoGA&N{C+8Wtu!K^Xil5UNTUfKQu$!XERxvpQcGgon2@elwt zLQ>@#n_c3JS`e(pj5_g}a2SFUI1Uq;aMr5K9I9ADOM{Y_F65^^q6f@D_sDN4duNdO z0*@GA@5<#p%w~3YX-=FNjaEj=i8KB(sD9c##8!;Y90B!q1MwpGH^H#lRU~;M@^$M+ z7K_arZB{fY*243^TTAkf?=iDi$u=TdBec^+D`L;4#-0%gVQigw!-~5 zx%yvP$8RafGfmuF7I%S^C72ngb~s-@*%Pt{bgJb{dCI2&xXHu}PY{>w+DaR7`Y+zL zQ+Ps%7@YljEE^=vgzhaARu%~r!sUCau!!KKw6^{S!yTh~iw(W{Gun8-(UgfVo+p^PoO3fx z-kGu!;x+$N#+g%KUtOF=Bcd@0R3`ZoqELX399;a_B*A9W@nX^FCO3inO{?N)^<1~| zmkKF9$^A|(4*8*SrdW5Ef>4M#tf{ebhE75iY8arK?|7Rw&C7c$mHlDo1f8M_tiE!~ zmmvUqc2)GjIAW0E^Kk^Xcvya`nwQa#3D`p^Qq{28Qi$O_CF0C$$bwtYKm)OEmzY=7 z46@y%!ysQp_xZgjd2$>^bjjpo3o2(|;Uir#uJOP6zh z8%M#?KwgsK6&R9EL9@H58ec1;6F7-LKT0QvbcQB zm7G*a!0`Yb$vnKkGc+Gn%Bs3caxUa?s!C*|&z^gEH%N;TH9uWKsO;aE`;o11J#GE! z|Do?UB9d^~oL|K?(%$-(vHW9I6V8HjFfe0FQ+d3hYmrY3?@-~DlS}gVd2QpKR&D_3 z3n^Cqn>2#i)R6;?Ea%9b6Cr~57sJqL8Bo%ICvOo%W8!7P=EsDu_dfVe{D-xF2!;9) zEt$IBov}m;q+eHJr_vUN-?6kbZ;E@Oe(l83sXTE+BJu(%NbjEvu+MFqtX(bS?e|CsUc(>>)Qlx$<489uufIf!MvOj zQS?IUd7~ysX1}96Ja!Ux%_1mVAd=It{I(Xpo0p^s3Bi2lnD!eVZ3P8?>%}l;)0CVu zh@>q221!LtNgILS&H0^pyFn-KCOD4}2r%v(51Q6mUwpfr9?&JKZsoXfLhHt+C%Y?C zo$V)Lr?37Kk2K*!Wm*?|qBK%Vxnf%M%7#R27+JnDNU{j~3#(^U!kykN!KO(&TBI5J z^UJ$OJtJCyY8W6ppt`X>B8>1wc|EBkEPKboY!{41BDIyTu%U^hF7yX-R4V4%4dLZP zd$#Q#vf@dLo=kZTFJkN;gV{fG*77JB$JO5_E5OV-JpQgrl4=$}QgVgf@Cu3C}6`3*GMGxCIq*~W!tk(*&R4_%r-*><|^lVdj-&Cp<}|mzZAb|f$STV^mba0Z~>`; z?Y;XE4;v0py$Fjdny9#|VO+kbWpY?}!bQ*#F8&aJcqbDp+@U9LVS|m(S2eoBvsj&s z`pYSJ<90L$X=6fJdzA^;Jn?bJl3f_=^^!=; zhm!x87Tk6vzW|5qv(&M_pn5qD(}Rc=kmBT8)<^8&i4++|h;+eB&Zbf-S2j2{rM6O~s$z_doF+K3i z=;mO;wiB}Bxff8w2Oi0#cARA9F~>>I^{6$`6?QM;F((xc6=Ew}bYtxNI%zwRPgAlH z&cO+6G~S%^To4{^E})TvZx*I1K{$@wZhALI{{mVghwY>x%j(JDue)o34b@eMmY}Hf zwPXKDOJRgl2k6gM*d-6PoZIlTJ2*X(-z`Gu3}B`|(pfh$jOUPHakjjAd~HM^Cf49d zfi>Zc;$C8;_GoHwx_W~L>KJN3HBlb7h2qWgrKxP1mYdB!5v3L5bTY;^1y#MpeUaEH zTfSp^n_(OI6j(Kg2l*5X7aftxt8>pJWU3_0jY^U9$p!+6LksY!$ z!GiQC<#xn2#-ONeV+@la=Gu@UlBv19NoLP2c2|Vvii?UgtWVj;OK+^Nt*0R>zyOt( z#j#0Fia5+f6Wips7{S<{&wP2as6G0o(%G#Ox%z_P8`>CCu=dLIqq1h09k=3vAo<&*Z^$*$O{vIA)Bz;QOl{r7{Yij*3icWj7D6v8bzQIuXiPasm+nJjlipH9q zv-42Uqc^moGFc}4SAHb+JKc~$PS0akZLo33-$~HXuwnF+Js2!LvR0?!xdAo)(L zpxhP)m7zReffFHyHwW^TT`cbl{APr*gHtrzCc+wwu*b@mA|Oo!WJ|b-$J2~BJ=XrJ zH^WHMvBA46T1=!VWC2EPlTtJ*FlKA_AJ8}@tjeG1k(;zpawWUu2aWg^O+EPOQ{Wbe zIR^)0k!fGo)zuiRE)K$N<-npOr@~OHWDbfzzPQ0sZfSaWPphm3af~IV*BXmXL8jfu z8*K*RNJ?QpN7r&tyYn4`C6L989O~jUjKz*Pk(ZIIlG?AuLhfJ<)K=OFn1dj3$=)7S z*yTofs8)qMvT`-VQFR+J!S3%94SA2x15ggciUibc6a7P86~$f?42Wej{ifDML-C_& z`D0&i-F7*9j-A2dT1(ua8N&75_6hp-)Guz@5o5hA8X$d_NfI0{2K7}JIUTe^&@akN z3gLLZ1C&%G%9QNrGL7;Q?=kERw%W(ydyE67pLSgAJ6HshxQG<{yhE)vz!EARaQYSh zq|-r@Sx0FA0!vL{MN3xSCk1vid{u1KeoJ3SI}`he&_zmmgG2L^NrsBH+M^+tC&JK0 ze@Mcb*w5^#D9@54BU2^@1#4xSF+wj|9TD&Lp-Vl#U+T>yZ3HkE@xr$AB`{-ZKDPQx zTzrQ|>k8J=7NIktf>VfEsL~meaqc2iZvLBV%gheQ@8W6+)NpbMixZXR5$HC)_wu{H zuRbZp=eoX_4YZ}xy zm+1-y20h%Q?lOHI-ea+SaU@K7E2Wy%yZqpeYG@96gCe10D!%AckMhb6uRGK;K{=eX zgqYzYul|H|=vDga&EdI5RzWdo5u72LRjds{T8?nKrDqhqTQ-_LLn?;q@sOE*4D@{^$BFH>x>OCy=#UTlH$&N^SQ{QS0uIH|vn5!1n`p^`@| zG{8#Qs*o}8i0u9~X$Dq_33eu7R8Wepe2CZ{(+ByA{g=B*V{~|wD&WV|Y-%1rnc}4r zxX{vpXq9V>it|+|CJ5F1GSX&;-9@tVO^qsa?~;?>UY%~~QPj;--A!vPe?)T)dvN4i zwmd85Zygjv>~Gc2-|GVQhxg_-U(ki9u+cluBLJRY?QksCL;rbgW0#+QEf@L}^(0r2AEfE`0zcG9ONSDy~*8;E9_ETT58p9a|c$v{G)Bwx?+cS0F+B z{$6Pg)FZY|gePgoqF&=8a4*wBoUy^voDFi(JduK)^m*K8xr_ru0}OAq0IVMz5dE6b z=+w%WeQ7;6L1MKx<#7-cUcPYtl1w5ts$;aJo1M9Z-ZmtG`TL}poLd?;lZW9Z4`_hB zj#1P~jFNu1+886)i`>(XG^&nhHlpLPck=%vxBNf(Y#$SnF zC4K%;mKon;0qY!h?X9r*Z0DZC@WXzPhx*=z0+WS$c&5jr z-m!RjzdUfhY^tE;FozDX`a|l(iNC7KggCOT;bO5@02IH`CqC8`LvAi+oT4EwY-Krl z42YUufnDyhIChO7dWP<;wnAnQB*W$VL@~1=$G;l&Ip81Z)h2k|)&q{9u&J)x&@6ZF z&fFs5gj`*{O>NFca#yxBHGLSL=VSo@kfOE-LtLpw3xq+!q;l%)v}|%6!>gaW^Yz_C zCzRg30s!72tY{REv8}55rU%L(997OuP#EJ<)ANSz`IvYHisp^1T2m6b?qUh>7STe9 z-pw2S>0B8Q&3^aB+4Ge0Y>Ajc0h}>)rx+MFA_oAYVstndV_BwxV~cJw?C?wI8#^`c zAfn>Pv5{-E+qE=n!8iTz7#^wy=!v8!1|}!AdL6as*NO6{E8e#V*rVDr_48L4qt&yY)S`p1X5TgGA+q`QxK&#myAftJ6GZ5U?-3YhkHVJGSK?+*3EE6!v zn$DRK0mZIAIAum#HmJ-~+MA?TkD}cS?)#TI0GRCj%3$GD`llA1d^FWs7g;}tHdGgh z_O9Tl)FeKL(gP|r753=;$d(SJ{&ESq`!UqA+7%&J*oZ^W^Z-!fBy#hFCi6WS8?@)_ zyBM;UP6s)g@fuGBtDm^Sx03Dv=6!YFF-P$IG5#6;(Zgl3;7Q$*q+P?ZP`4bGjy literal 0 HcmV?d00001 From 5402210536dca849a4ead9f91ac097ff5de20e24 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Apr 2023 06:33:52 -0700 Subject: [PATCH 387/551] chore(deps): bump regex from 1.7.3 to 1.8.0 (#411) Bumps [regex](https://github.com/rust-lang/regex) from 1.7.3 to 1.8.0. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/commits) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0027296e..bf961a61e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -915,9 +915,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "ac6cf59af1067a3fb53fbe5c88c053764e930f932be1d71d3ffe032cbe147f59" dependencies = [ "aho-corasick", "memchr", @@ -926,9 +926,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.29" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "b6868896879ba532248f33598de5181522d8b3d9d724dfd230911e1a7d4822f5" [[package]] name = "ring" From 725586371427446381442a64d706b15bc39c338f Mon Sep 17 00:00:00 2001 From: Shawn <18649508+NOBLES5E@users.noreply.github.com> Date: Wed, 26 Apr 2023 11:28:54 -0400 Subject: [PATCH 388/551] fix typo (#415) chore: typo --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 00c0f0680..5ff09ee15 100644 --- a/src/config.rs +++ b/src/config.rs @@ -588,7 +588,7 @@ impl Shard { if primary_count > 1 { error!( - "Shard {} has more than on primary configured", + "Shard {} has more than one primary configured", self.database ); return Err(Error::BadConfig); From 8d2ecec5fdff106fe0e5f07da3e4bbd686af11db Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 26 Apr 2023 16:33:26 -0700 Subject: [PATCH 389/551] Add more pool settings (#416) * Add some pool settings * fmt --- CONFIG.md | 8 ++++ pgcat.toml | 5 +++ src/auth_passthrough.rs | 2 + src/config.rs | 91 +++++++++++++++++++++++++++++++++++++++-- src/pool.rs | 53 +++++++++++++++++------- 5 files changed, 141 insertions(+), 18 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 72d5683a1..1a05a7760 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -49,6 +49,14 @@ default: 30000 # milliseconds How long an idle connection with a server is left open (ms). +### server_lifetime +``` +path: general.server_lifetime +default: 86400000 # 24 hours +``` + +Max connection lifetime before it's closed, even if actively used. + ### idle_client_in_transaction_timeout ``` path: general.idle_client_in_transaction_timeout diff --git a/pgcat.toml b/pgcat.toml index 6cb9a2991..9203cb603 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -23,6 +23,9 @@ connect_timeout = 5000 # milliseconds # How long an idle connection with a server is left open (ms). idle_timeout = 30000 # milliseconds +# Max connection lifetime before it's closed, even if actively used. +server_lifetime = 86400000 # 24 hours + # How long a client is allowed to be idle while in a transaction (ms). idle_client_in_transaction_timeout = 0 # milliseconds @@ -206,6 +209,8 @@ sharding_function = "pg_bigint_hash" username = "simple_user" password = "simple_user" pool_size = 5 +min_pool_size = 3 +server_lifetime = 60000 statement_timeout = 0 [pools.simple_db.shards.0] diff --git a/src/auth_passthrough.rs b/src/auth_passthrough.rs index 4e9362623..fc0f6dc69 100644 --- a/src/auth_passthrough.rs +++ b/src/auth_passthrough.rs @@ -77,6 +77,8 @@ impl AuthPassthrough { pool_size: 1, statement_timeout: 0, pool_mode: None, + server_lifetime: None, + min_pool_size: None, }; let user = &address.username; diff --git a/src/config.rs b/src/config.rs index 5ff09ee15..d822486dc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -181,7 +181,9 @@ pub struct User { pub server_username: Option, pub server_password: Option, pub pool_size: u32, + pub min_pool_size: Option, pub pool_mode: Option, + pub server_lifetime: Option, #[serde(default)] // 0 pub statement_timeout: u64, } @@ -194,12 +196,34 @@ impl Default for User { server_username: None, server_password: None, pool_size: 15, + min_pool_size: None, statement_timeout: 0, pool_mode: None, + server_lifetime: None, } } } +impl User { + fn validate(&self) -> Result<(), Error> { + match self.min_pool_size { + Some(min_pool_size) => { + if min_pool_size > self.pool_size { + error!( + "min_pool_size of {} cannot be larger than pool_size of {}", + min_pool_size, self.pool_size + ); + return Err(Error::BadConfig); + } + } + + None => (), + }; + + Ok(()) + } +} + /// General configuration. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct General { @@ -246,6 +270,9 @@ pub struct General { #[serde(default = "General::default_idle_client_in_transaction_timeout")] pub idle_client_in_transaction_timeout: u64, + #[serde(default = "General::default_server_lifetime")] + pub server_lifetime: u64, + #[serde(default = "General::default_worker_threads")] pub worker_threads: usize, @@ -271,6 +298,10 @@ impl General { 5432 } + pub fn default_server_lifetime() -> u64 { + 1000 * 60 * 60 * 24 // 24 hours + } + pub fn default_connect_timeout() -> u64 { 1000 } @@ -347,6 +378,7 @@ impl Default for General { auth_query: None, auth_query_user: None, auth_query_password: None, + server_lifetime: 1000 * 3600 * 24, // 24 hours, } } } @@ -411,6 +443,8 @@ pub struct Pool { pub idle_timeout: Option, + pub server_lifetime: Option, + pub sharding_function: ShardingFunction, #[serde(default = "Pool::default_automatic_sharding_key")] @@ -515,6 +549,10 @@ impl Pool { None => None, }; + for (_, user) in &self.users { + user.validate()?; + } + Ok(()) } } @@ -539,6 +577,7 @@ impl Default for Pool { auth_query: None, auth_query_user: None, auth_query_password: None, + server_lifetime: None, } } } @@ -791,6 +830,10 @@ impl Config { ); info!("Shutdown timeout: {}ms", self.general.shutdown_timeout); info!("Healthcheck delay: {}ms", self.general.healthcheck_delay); + info!( + "Default max server lifetime: {}ms", + self.general.server_lifetime + ); match self.general.tls_certificate.clone() { Some(tls_certificate) => { info!("TLS certificate: {}", tls_certificate); @@ -867,12 +910,26 @@ impl Config { pool_name, pool_config.users.len() ); + info!( + "[pool: {}] Max server lifetime: {}", + pool_name, + match pool_config.server_lifetime { + Some(server_lifetime) => format!("{}ms", server_lifetime), + None => "default".to_string(), + } + ); for user in &pool_config.users { info!( "[pool: {}][user: {}] Pool size: {}", pool_name, user.1.username, user.1.pool_size, ); + info!( + "[pool: {}][user: {}] Minimum pool size: {}", + pool_name, + user.1.username, + user.1.min_pool_size.unwrap_or(0) + ); info!( "[pool: {}][user: {}] Statement timeout: {}", pool_name, user.1.username, user.1.statement_timeout @@ -886,6 +943,15 @@ impl Config { None => pool_config.pool_mode.to_string(), } ); + info!( + "[pool: {}][user: {}] Max server lifetime: {}", + pool_name, + user.1.username, + match user.1.server_lifetime { + Some(server_lifetime) => format!("{}ms", server_lifetime), + None => "default".to_string(), + } + ); } } } @@ -896,7 +962,13 @@ impl Config { && (self.general.auth_query_user.is_none() || self.general.auth_query_password.is_none()) { - error!("If auth_query is specified, you need to provide a value for `auth_query_user`, `auth_query_password`"); + error!( + "If auth_query is specified, \ + you need to provide a value \ + for `auth_query_user`, \ + `auth_query_password`" + ); + return Err(Error::BadConfig); } @@ -904,7 +976,14 @@ impl Config { if pool.auth_query.is_some() && (pool.auth_query_user.is_none() || pool.auth_query_password.is_none()) { - error!("Error in pool {{ {} }}. If auth_query is specified, you need to provide a value for `auth_query_user`, `auth_query_password`", name); + error!( + "Error in pool {{ {} }}. \ + If auth_query is specified, you need \ + to provide a value for `auth_query_user`, \ + `auth_query_password`", + name + ); + return Err(Error::BadConfig); } @@ -914,7 +993,13 @@ impl Config { || pool.auth_query_user.is_none()) && user_data.password.is_none() { - error!("Error in pool {{ {} }}. You have to specify a user password for every pool if auth_query is not specified", name); + error!( + "Error in pool {{ {} }}. \ + You have to specify a user password \ + for every pool if auth_query is not specified", + name + ); + return Err(Error::BadConfig); } } diff --git a/src/pool.rs b/src/pool.rs index 7f8e41c0a..8ec886041 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -311,21 +311,34 @@ impl ConnectionPool { if let Some(apt) = &auth_passthrough { match apt.fetch_hash(&address).await { - Ok(ok) => { - if let Some(ref pool_auth_hash_value) = *(pool_auth_hash.read()) { - if ok != *pool_auth_hash_value { - warn!("Hash is not the same across shards of the same pool, client auth will \ - be done using last obtained hash. Server: {}:{}, Database: {}", server.host, server.port, shard.database); - } - } - debug!("Hash obtained for {:?}", address); - { - let mut pool_auth_hash = pool_auth_hash.write(); - *pool_auth_hash = Some(ok.clone()); - } - }, - Err(err) => warn!("Could not obtain password hashes using auth_query config, ignoring. Error: {:?}", err), - } + Ok(ok) => { + if let Some(ref pool_auth_hash_value) = *(pool_auth_hash.read()) + { + if ok != *pool_auth_hash_value { + warn!( + "Hash is not the same across shards \ + of the same pool, client auth will \ + be done using last obtained hash. \ + Server: {}:{}, Database: {}", + server.host, server.port, shard.database, + ); + } + } + + debug!("Hash obtained for {:?}", address); + + { + let mut pool_auth_hash = pool_auth_hash.write(); + *pool_auth_hash = Some(ok.clone()); + } + } + Err(err) => warn!( + "Could not obtain password hashes \ + using auth_query config, ignoring. \ + Error: {:?}", + err, + ), + } } let manager = ServerPool::new( @@ -347,10 +360,20 @@ impl ConnectionPool { None => config.general.idle_timeout, }; + let server_lifetime = match user.server_lifetime { + Some(server_lifetime) => server_lifetime, + None => match pool_config.server_lifetime { + Some(server_lifetime) => server_lifetime, + None => config.general.server_lifetime, + }, + }; + let pool = Pool::builder() .max_size(user.pool_size) + .min_idle(user.min_pool_size) .connection_timeout(std::time::Duration::from_millis(connect_timeout)) .idle_timeout(Some(std::time::Duration::from_millis(idle_timeout))) + .max_lifetime(Some(std::time::Duration::from_millis(server_lifetime))) .test_on_check_out(false) .build(manager) .await From 7a2aeed57432da313c03f7a4d632295cb9f25b8a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 30 Apr 2023 09:41:46 -0700 Subject: [PATCH 390/551] Server TLS (#417) * Server TLS * Finish up TLS * thats it * diff * remove dead code * maybe? * dirty shutdown * skip flakey test * remove unused error * fetch config once --- Cargo.lock | 32 ++++++ Cargo.toml | 3 + pgcat.toml | 10 +- src/client.rs | 13 ++- src/config.rs | 14 +++ src/messages.rs | 43 +++++++- src/pool.rs | 3 +- src/server.rs | 201 ++++++++++++++++++++++++++++++++----- src/tls.rs | 23 ++++- tests/ruby/mirrors_spec.rb | 2 +- 10 files changed, 311 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf961a61e..7991667ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -762,9 +762,11 @@ dependencies = [ "once_cell", "parking_lot", "phf", + "pin-project", "postgres-protocol", "rand", "regex", + "rustls", "rustls-pemfile", "serde", "serde_derive", @@ -776,6 +778,7 @@ dependencies = [ "tokio", "tokio-rustls", "toml", + "webpki-roots", ] [[package]] @@ -820,6 +823,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1446,6 +1469,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa54963694b65584e170cf5dc46aeb4dcaa5584e652ff5f3952e56d66aff0125" +dependencies = [ + "rustls-webpki", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index a5573518e..28e94a6d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,9 @@ nix = "0.26.2" atomic_enum = "0.2.0" postgres-protocol = "0.6.5" fallible-iterator = "0.2" +pin-project = "1" +webpki-roots = "0.23" +rustls = { version = "0.21", features = ["dangerous_configuration"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/pgcat.toml b/pgcat.toml index 9203cb603..df2ba7155 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -61,9 +61,15 @@ tcp_keepalives_count = 5 tcp_keepalives_interval = 5 # Path to TLS Certificate file to use for TLS connections -# tls_certificate = "server.cert" +# tls_certificate = ".circleci/server.cert" # Path to TLS private key file to use for TLS connections -# tls_private_key = "server.key" +# tls_private_key = ".circleci/server.key" + +# Enable/disable server TLS +server_tls = false + +# Verify server certificate is completely authentic. +verify_server_certificate = false # User name to access the virtual administrative database (pgbouncer or pgcat) # Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. diff --git a/src/client.rs b/src/client.rs index 5098ec6f5..efde75544 100644 --- a/src/client.rs +++ b/src/client.rs @@ -539,6 +539,7 @@ where Some(md5_hash_password(username, password, &salt)) } else { if !get_config().is_auth_query_configured() { + wrong_password(&mut write, username).await?; return Err(Error::ClientAuthImpossible(username.into())); } @@ -565,6 +566,8 @@ where } Err(err) => { + wrong_password(&mut write, username).await?; + return Err(Error::ClientAuthPassthroughError( err.to_string(), client_identifier, @@ -587,7 +590,15 @@ where client_identifier ); - let fetched_hash = refetch_auth_hash(&pool).await?; + let fetched_hash = match refetch_auth_hash(&pool).await { + Ok(fetched_hash) => fetched_hash, + Err(err) => { + wrong_password(&mut write, username).await?; + + return Err(err); + } + }; + let new_password_hash = md5_hash_second_pass(&fetched_hash, &salt); // Ok password changed in server an auth is possible. diff --git a/src/config.rs b/src/config.rs index d822486dc..4af7beda5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -281,6 +281,13 @@ pub struct General { pub tls_certificate: Option, pub tls_private_key: Option, + + #[serde(default)] // false + pub server_tls: bool, + + #[serde(default)] // false + pub verify_server_certificate: bool, + pub admin_username: String, pub admin_password: String, @@ -373,6 +380,8 @@ impl Default for General { autoreload: None, tls_certificate: None, tls_private_key: None, + server_tls: false, + verify_server_certificate: false, admin_username: String::from("admin"), admin_password: String::from("admin"), auth_query: None, @@ -852,6 +861,11 @@ impl Config { info!("TLS support is disabled"); } }; + info!("Server TLS enabled: {}", self.general.server_tls); + info!( + "Server TLS certificate verification: {}", + self.general.verify_server_certificate + ); for (pool_name, pool_config) in &self.pools { // TODO: Make this output prettier (maybe a table?) diff --git a/src/messages.rs b/src/messages.rs index ba4818cec..0e980fe6b 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -116,7 +116,10 @@ where /// Send the startup packet the server. We're pretending we're a Pg client. /// This tells the server which user we are and what database we want. -pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Result<(), Error> { +pub async fn startup(stream: &mut S, user: &str, database: &str) -> Result<(), Error> +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ let mut bytes = BytesMut::with_capacity(25); bytes.put_i32(196608); // Protocol number @@ -150,6 +153,21 @@ pub async fn startup(stream: &mut TcpStream, user: &str, database: &str) -> Resu } } +pub async fn ssl_request(stream: &mut TcpStream) -> Result<(), Error> { + let mut bytes = BytesMut::with_capacity(12); + + bytes.put_i32(8); + bytes.put_i32(80877103); + + match stream.write_all(&bytes).await { + Ok(_) => Ok(()), + Err(err) => Err(Error::SocketError(format!( + "Error writing SSLRequest to server socket - Error: {:?}", + err + ))), + } +} + /// Parse the params the server sends as a key/value format. pub fn parse_params(mut bytes: BytesMut) -> Result, Error> { let mut result = HashMap::new(); @@ -505,6 +523,29 @@ where } } +pub async fn write_all_flush(stream: &mut S, buf: &[u8]) -> Result<(), Error> +where + S: tokio::io::AsyncWrite + std::marker::Unpin, +{ + match stream.write_all(buf).await { + Ok(_) => match stream.flush().await { + Ok(_) => Ok(()), + Err(err) => { + return Err(Error::SocketError(format!( + "Error flushing socket - Error: {:?}", + err + ))) + } + }, + Err(err) => { + return Err(Error::SocketError(format!( + "Error writing to socket - Error: {:?}", + err + ))) + } + } +} + /// Read a complete message from the socket. pub async fn read_message(stream: &mut S) -> Result where diff --git a/src/pool.rs b/src/pool.rs index 8ec886041..ee8de446f 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -376,8 +376,7 @@ impl ConnectionPool { .max_lifetime(Some(std::time::Duration::from_millis(server_lifetime))) .test_on_check_out(false) .build(manager) - .await - .unwrap(); + .await?; pools.push(pool); servers.push(address); diff --git a/src/server.rs b/src/server.rs index 84bed6ccf..5bcd5fb99 100644 --- a/src/server.rs +++ b/src/server.rs @@ -9,13 +9,12 @@ use std::collections::HashMap; use std::io::Read; use std::sync::Arc; use std::time::SystemTime; -use tokio::io::{AsyncReadExt, BufReader}; -use tokio::net::{ - tcp::{OwnedReadHalf, OwnedWriteHalf}, - TcpStream, -}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, BufStream}; +use tokio::net::TcpStream; +use tokio_rustls::rustls::{OwnedTrustAnchor, RootCertStore}; +use tokio_rustls::{client::TlsStream, TlsConnector}; -use crate::config::{Address, User}; +use crate::config::{get_config, Address, User}; use crate::constants::*; use crate::errors::{Error, ServerIdentifier}; use crate::messages::*; @@ -23,6 +22,84 @@ use crate::mirrors::MirroringManager; use crate::pool::ClientServerMap; use crate::scram::ScramSha256; use crate::stats::ServerStats; +use std::io::Write; + +use pin_project::pin_project; + +#[pin_project(project = SteamInnerProj)] +pub enum StreamInner { + Plain { + #[pin] + stream: TcpStream, + }, + Tls { + #[pin] + stream: TlsStream, + }, +} + +impl AsyncWrite for StreamInner { + fn poll_write( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + let this = self.project(); + match this { + SteamInnerProj::Tls { stream } => stream.poll_write(cx, buf), + SteamInnerProj::Plain { stream } => stream.poll_write(cx, buf), + } + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let this = self.project(); + match this { + SteamInnerProj::Tls { stream } => stream.poll_flush(cx), + SteamInnerProj::Plain { stream } => stream.poll_flush(cx), + } + } + + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let this = self.project(); + match this { + SteamInnerProj::Tls { stream } => stream.poll_shutdown(cx), + SteamInnerProj::Plain { stream } => stream.poll_shutdown(cx), + } + } +} + +impl AsyncRead for StreamInner { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.project(); + match this { + SteamInnerProj::Tls { stream } => stream.poll_read(cx, buf), + SteamInnerProj::Plain { stream } => stream.poll_read(cx, buf), + } + } +} + +impl StreamInner { + pub fn try_write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + StreamInner::Tls { stream } => { + let r = stream.get_mut(); + let mut w = r.1.writer(); + w.write(buf) + } + StreamInner::Plain { stream } => stream.try_write(buf), + } + } +} /// Server state. pub struct Server { @@ -30,11 +107,8 @@ pub struct Server { /// port, e.g. 5432, and role, e.g. primary or replica. address: Address, - /// Buffered read socket. - read: BufReader, - - /// Unbuffered write socket (our client code buffers). - write: OwnedWriteHalf, + /// Server TCP connection. + stream: BufStream, /// Our server response buffer. We buffer data before we give it to the client. buffer: BytesMut, @@ -98,8 +172,88 @@ impl Server { ))); } }; + + // TCP timeouts. configure_socket(&stream); + let config = get_config(); + + let mut stream = if config.general.server_tls { + // Request a TLS connection + ssl_request(&mut stream).await?; + + let response = match stream.read_u8().await { + Ok(response) => response as char, + Err(err) => { + return Err(Error::SocketError(format!( + "Server socket error: {:?}", + err + ))) + } + }; + + match response { + // Server supports TLS + 'S' => { + debug!("Connecting to server using TLS"); + + let mut root_store = RootCertStore::empty(); + root_store.add_server_trust_anchors( + webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| { + OwnedTrustAnchor::from_subject_spki_name_constraints( + ta.subject, + ta.spki, + ta.name_constraints, + ) + }), + ); + + let mut tls_config = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(root_store) + .with_no_client_auth(); + + // Equivalent to sslmode=prefer which is fine most places. + // If you want verify-full, change `verify_server_certificate` to true. + if !config.general.verify_server_certificate { + let mut dangerous = tls_config.dangerous(); + dangerous.set_certificate_verifier(Arc::new( + crate::tls::NoCertificateVerification {}, + )); + } + + let connector = TlsConnector::from(Arc::new(tls_config)); + let stream = match connector + .connect(address.host.as_str().try_into().unwrap(), stream) + .await + { + Ok(stream) => stream, + Err(err) => { + return Err(Error::SocketError(format!("Server TLS error: {:?}", err))) + } + }; + + StreamInner::Tls { stream } + } + + // Server does not support TLS + 'N' => StreamInner::Plain { stream }, + + // Something else? + m => { + return Err(Error::SocketError(format!( + "Unknown message: {}", + m as char + ))); + } + } + } else { + StreamInner::Plain { stream } + }; + + // let (read, write) = split(stream); + // let (mut read, mut write) = (ReadInner::Plain { stream: read }, WriteInner::Plain { stream: write }); + trace!("Sending StartupMessage"); // StartupMessage @@ -245,7 +399,7 @@ impl Server { let sasl_type = String::from_utf8_lossy(&sasl_auth[..sasl_len - 2]); - if sasl_type == SCRAM_SHA_256 { + if sasl_type.contains(SCRAM_SHA_256) { debug!("Using {}", SCRAM_SHA_256); // Generate client message. @@ -268,7 +422,7 @@ impl Server { res.put_i32(sasl_response.len() as i32); res.put(sasl_response); - write_all(&mut stream, res).await?; + write_all_flush(&mut stream, &res).await?; } else { error!("Unsupported SCRAM version: {}", sasl_type); return Err(Error::ServerError); @@ -299,7 +453,7 @@ impl Server { res.put_i32(4 + sasl_response.len() as i32); res.put(sasl_response); - write_all(&mut stream, res).await?; + write_all_flush(&mut stream, &res).await?; } SASL_FINAL => { @@ -443,12 +597,9 @@ impl Server { } }; - let (read, write) = stream.into_split(); - let mut server = Server { address: address.clone(), - read: BufReader::new(read), - write, + stream: BufStream::new(stream), buffer: BytesMut::with_capacity(8196), server_info, process_id, @@ -515,7 +666,7 @@ impl Server { bytes.put_i32(process_id); bytes.put_i32(secret_key); - write_all(&mut stream, bytes).await + write_all_flush(&mut stream, &bytes).await } /// Send messages to the server from the client. @@ -523,7 +674,7 @@ impl Server { self.mirror_send(messages); self.stats().data_sent(messages.len()); - match write_all_half(&mut self.write, messages).await { + match write_all_flush(&mut self.stream, &messages).await { Ok(_) => { // Successfully sent to server self.last_activity = SystemTime::now(); @@ -542,7 +693,7 @@ impl Server { /// in order to receive all data the server has to offer. pub async fn recv(&mut self) -> Result { loop { - let mut message = match read_message(&mut self.read).await { + let mut message = match read_message(&mut self.stream).await { Ok(message) => message, Err(err) => { error!("Terminating server because of: {:?}", err); @@ -935,13 +1086,13 @@ impl Drop for Server { // Update statistics self.stats.disconnect(); - let mut bytes = BytesMut::with_capacity(4); + let mut bytes = BytesMut::with_capacity(5); bytes.put_u8(b'X'); bytes.put_i32(4); - match self.write.try_write(&bytes) { - Ok(_) => (), - Err(_) => debug!("Dirty shutdown"), + match self.stream.get_mut().try_write(&bytes) { + Ok(5) => (), + _ => debug!("Dirty shutdown"), }; // Should not matter. diff --git a/src/tls.rs b/src/tls.rs index fbfbae759..6c4a7f5b3 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -4,7 +4,12 @@ use rustls_pemfile::{certs, read_one, Item}; use std::iter; use std::path::Path; use std::sync::Arc; -use tokio_rustls::rustls::{self, Certificate, PrivateKey}; +use std::time::SystemTime; +use tokio_rustls::rustls::{ + self, + client::{ServerCertVerified, ServerCertVerifier}, + Certificate, PrivateKey, ServerName, +}; use tokio_rustls::TlsAcceptor; use crate::config::get_config; @@ -64,3 +69,19 @@ impl Tls { }) } } + +pub struct NoCertificateVerification; + +impl ServerCertVerifier for NoCertificateVerification { + fn verify_server_cert( + &self, + _end_entity: &Certificate, + _intermediates: &[Certificate], + _server_name: &ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: SystemTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } +} diff --git a/tests/ruby/mirrors_spec.rb b/tests/ruby/mirrors_spec.rb index 801df28cb..898d0d718 100644 --- a/tests/ruby/mirrors_spec.rb +++ b/tests/ruby/mirrors_spec.rb @@ -25,7 +25,7 @@ processes.pgcat.shutdown end - it "can mirror a query" do + xit "can mirror a query" do conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) runs = 15 runs.times { conn.async_exec("SELECT 1 + 2") } From 29304b4397538a28b072e76b592de64a6f8a68b4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 30 Apr 2023 09:44:25 -0700 Subject: [PATCH 391/551] Readme update (#418) * Readme update * m * wording --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4dd752460..ae310cded 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load bal | Failover | **Stable** | Queries are automatically rerouted around broken replicas, validated by regular health checks. | | Admin database statistics | **Stable** | Pooler statistics and administration via the `pgbouncer` and `pgcat` databases. | | Prometheus statistics | **Stable** | Statistics are reported via a HTTP endpoint for Prometheus. | -| Client TLS | **Stable** | Clients can connect to the pooler using TLS/SSL. | +| SSL/TLS | **Stable** | Clients can connect to the pooler using TLS. Pooler can connect to Postgres servers using TLS. | | Client/Server authentication | **Stable** | Clients can connect using MD5 authentication, supported by `libpq` and all Postgres client drivers. PgCat can connect to Postgres using MD5 and SCRAM-SHA-256. | | Live configuration reloading | **Stable** | Identical to PgBouncer; all settings can be reloaded dynamically (except `host` and `port`). | | Auth passthrough | **Stable** | MD5 password authentication can be configured to use an `auth_query` so no cleartext passwords are needed in the config file.| From 6a4562434d23d0a276c805da0b9508bb2fb67430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Tue, 2 May 2023 10:26:40 +0200 Subject: [PATCH 392/551] Add dns_cache for server addresses as in pgbouncer (#249) * Add dns_cache so server addresses are cached and invalidated when DNS changes. Adds a module to deal with dns_cache feature. It's main struct is CachedResolver, which is a simple thread safe hostname <-> Ips cache with the ability to refresh resolutions every `dns_max_ttl` seconds. This way, a client can check whether its ip address has changed. * Allow reloading dns cached * Add documentation for dns_cached --- CONFIG.md | 16 ++ Cargo.lock | 288 +++++++++++++++++++++++++++++++++ Cargo.toml | 2 + pgcat.toml | 8 + src/config.rs | 17 ++ src/dns_cache.rs | 410 +++++++++++++++++++++++++++++++++++++++++++++++ src/errors.rs | 1 + src/lib.rs | 1 + src/main.rs | 12 +- src/server.rs | 42 ++++- 10 files changed, 794 insertions(+), 3 deletions(-) create mode 100644 src/dns_cache.rs diff --git a/CONFIG.md b/CONFIG.md index 1a05a7760..b36a190df 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -188,6 +188,22 @@ default: "admin_pass" Password to access the virtual administrative database +### dns_cache_enabled +``` +path: general.dns_cache_enabled +default: false +``` +When enabled, ip resolutions for server connections specified using hostnames will be cached +and checked for changes every `dns_max_ttl` seconds. If a change in the host resolution is found +old ip connections are closed (gracefully) and new connections will start using new ip. + +### dns_max_ttl +``` +path: general.dns_max_ttl +default: 30 +``` +Specifies how often (in seconds) cached ip addresses for servers are rechecked (see `dns_cache_enabled`). + ## `pools.` Section ### pool_mode diff --git a/Cargo.lock b/Cargo.lock index 7991667ef..7641e9c29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,27 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "async-trait" version = "0.1.68" @@ -212,6 +233,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "digest" version = "0.10.6" @@ -223,6 +250,18 @@ dependencies = [ "subtle", ] +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "env_logger" version = "0.10.0" @@ -275,6 +314,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures" version = "0.3.28" @@ -410,6 +458,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "hermit-abi" version = "0.2.6" @@ -434,6 +488,17 @@ dependencies = [ "digest", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + [[package]] name = "http" version = "0.2.9" @@ -522,6 +587,27 @@ dependencies = [ "cxx-build", ] +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "1.9.2" @@ -542,6 +628,24 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "ipconfig" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd302af1b90f2463a98fa5ad469fc212c8e3175a41c3068601bfa2727591c5be" +dependencies = [ + "socket2", + "widestring", + "winapi", + "winreg", +] + +[[package]] +name = "ipnet" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" + [[package]] name = "is-terminal" version = "0.4.4" @@ -589,6 +693,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.139" @@ -604,6 +714,12 @@ dependencies = [ "cc", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.1.4" @@ -629,6 +745,27 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "md-5" version = "0.10.5" @@ -737,6 +874,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + [[package]] name = "pgcat" version = "1.0.1" @@ -777,7 +920,9 @@ dependencies = [ "stringprep", "tokio", "tokio-rustls", + "tokio-test", "toml", + "trust-dns-resolver", "webpki-roots", ] @@ -888,6 +1033,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.26" @@ -953,6 +1104,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6868896879ba532248f33598de5181522d8b3d9d724dfd230911e1a7d4822f5" +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + [[package]] name = "ring" version = "0.16.20" @@ -1191,6 +1352,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "time" version = "0.1.45" @@ -1258,6 +1439,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-test" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" +dependencies = [ + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-util" version = "0.7.7" @@ -1320,9 +1525,21 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "tracing-core" version = "0.1.30" @@ -1332,6 +1549,51 @@ dependencies = [ "once_cell", ] +[[package]] +name = "trust-dns-proto" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.2.3", + "ipnet", + "lazy_static", + "rand", + "smallvec", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lazy_static", + "lru-cache", + "parking_lot", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "tracing", + "trust-dns-proto", +] + [[package]] name = "try-lock" version = "0.2.4" @@ -1377,6 +1639,17 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna 0.3.0", + "percent-encoding", +] + [[package]] name = "version_check" version = "0.9.4" @@ -1478,6 +1751,12 @@ dependencies = [ "rustls-webpki", ] +[[package]] +name = "widestring" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" + [[package]] name = "winapi" version = "0.3.9" @@ -1583,3 +1862,12 @@ checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658" dependencies = [ "memchr", ] + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/Cargo.toml b/Cargo.toml index 28e94a6d9..436c3dd54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ fallible-iterator = "0.2" pin-project = "1" webpki-roots = "0.23" rustls = { version = "0.21", features = ["dangerous_configuration"] } +trust-dns-resolver = "0.22.0" +tokio-test = "0.4.2" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/pgcat.toml b/pgcat.toml index df2ba7155..c844ce1fc 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -146,6 +146,14 @@ idle_timeout = 40000 # Connect timeout can be overwritten in the pool connect_timeout = 3000 +# When enabled, ip resolutions for server connections specified using hostnames will be cached +# and checked for changes every `dns_max_ttl` seconds. If a change in the host resolution is found +# old ip connections are closed (gracefully) and new connections will start using new ip. +# dns_cache_enabled = false + +# Specifies how often (in seconds) cached ip addresses for servers are rechecked (see `dns_cache_enabled`). +# dns_max_ttl = 30 + # User configs are structured as pool..users. # This section holds the credentials for users that may connect to this cluster [pools.sharded_db.users.0] diff --git a/src/config.rs b/src/config.rs index 4af7beda5..fd7d3912c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use tokio::fs::File; use tokio::io::AsyncReadExt; +use crate::dns_cache::CachedResolver; use crate::errors::Error; use crate::pool::{ClientServerMap, ConnectionPool}; use crate::sharding::ShardingFunction; @@ -255,6 +256,12 @@ pub struct General { #[serde(default)] // False pub log_client_disconnections: bool, + #[serde(default)] // False + pub dns_cache_enabled: bool, + + #[serde(default = "General::default_dns_max_ttl")] + pub dns_max_ttl: u64, + #[serde(default = "General::default_shutdown_timeout")] pub shutdown_timeout: u64, @@ -336,6 +343,10 @@ impl General { 60000 } + pub fn default_dns_max_ttl() -> u64 { + 30 + } + pub fn default_healthcheck_timeout() -> u64 { 1000 } @@ -378,6 +389,8 @@ impl Default for General { log_client_connections: false, log_client_disconnections: false, autoreload: None, + dns_cache_enabled: false, + dns_max_ttl: Self::default_dns_max_ttl(), tls_certificate: None, tls_private_key: None, server_tls: false, @@ -1119,6 +1132,10 @@ pub async fn reload_config(client_server_map: ClientServerMap) -> Result (), + Err(err) => error!("DNS cache reinitialization error: {:?}", err), + }; if old_config.pools != new_config.pools { info!("Pool configuration changed"); diff --git a/src/dns_cache.rs b/src/dns_cache.rs new file mode 100644 index 000000000..5c2be5dc7 --- /dev/null +++ b/src/dns_cache.rs @@ -0,0 +1,410 @@ +use crate::config::get_config; +use crate::errors::Error; +use arc_swap::ArcSwap; +use log::{debug, error, info, warn}; +use once_cell::sync::Lazy; +use std::collections::{HashMap, HashSet}; +use std::io; +use std::net::IpAddr; +use std::sync::Arc; +use std::sync::RwLock; +use tokio::time::{sleep, Duration}; +use trust_dns_resolver::error::{ResolveError, ResolveResult}; +use trust_dns_resolver::lookup_ip::LookupIp; +use trust_dns_resolver::TokioAsyncResolver; + +/// Cached Resolver Globally available +pub static CACHED_RESOLVER: Lazy> = + Lazy::new(|| ArcSwap::from_pointee(CachedResolver::default())); + +// Ip addressed are returned as a set of addresses +// so we can compare. +#[derive(Clone, PartialEq, Debug)] +pub struct AddrSet { + set: HashSet, +} + +impl AddrSet { + fn new() -> AddrSet { + AddrSet { + set: HashSet::new(), + } + } +} + +impl From for AddrSet { + fn from(lookup_ip: LookupIp) -> Self { + let mut addr_set = AddrSet::new(); + for address in lookup_ip.iter() { + addr_set.set.insert(address); + } + addr_set + } +} + +/// +/// A CachedResolver is a DNS resolution cache mechanism with customizable expiration time. +/// +/// The system works as follows: +/// +/// When a host is to be resolved, if we have not resolved it before, a new resolution is +/// executed and stored in the internal cache. Concurrently, every `dns_max_ttl` time, the +/// cache is refreshed. +/// +/// # Example: +/// +/// ``` +/// use pgcat::dns_cache::{CachedResolverConfig, CachedResolver}; +/// +/// # tokio_test::block_on(async { +/// let config = CachedResolverConfig::default(); +/// let resolver = CachedResolver::new(config, None).await.unwrap(); +/// let addrset = resolver.lookup_ip("www.example.com.").await.unwrap(); +/// # }) +/// ``` +/// +/// // Now the ip resolution is stored in local cache and subsequent +/// // calls will be returned from cache. Also, the cache is refreshed +/// // and updated every 10 seconds. +/// +/// // You can now check if an 'old' lookup differs from what it's currently +/// // store in cache by using `has_changed`. +/// resolver.has_changed("www.example.com.", addrset) +#[derive(Default)] +pub struct CachedResolver { + // The configuration of the cached_resolver. + config: CachedResolverConfig, + + // This is the hash that contains the hash. + data: Option>>, + + // The resolver to be used for DNS queries. + resolver: Option, + + // The RefreshLoop + refresh_loop: RwLock>>, +} + +/// +/// Configuration +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CachedResolverConfig { + /// Amount of time in secods that a resolved dns address is considered stale. + dns_max_ttl: u64, + + /// Enabled or disabled? (this is so we can reload config) + enabled: bool, +} + +impl CachedResolverConfig { + fn new(dns_max_ttl: u64, enabled: bool) -> Self { + CachedResolverConfig { + dns_max_ttl, + enabled, + } + } +} + +impl From for CachedResolverConfig { + fn from(config: crate::config::Config) -> Self { + CachedResolverConfig::new(config.general.dns_max_ttl, config.general.dns_cache_enabled) + } +} + +impl CachedResolver { + /// + /// Returns a new Arc based on passed configuration. + /// It also starts the loop that will refresh cache entries. + /// + /// # Arguments: + /// + /// * `config` - The `CachedResolverConfig` to be used to create the resolver. + /// + /// # Example: + /// + /// ``` + /// use pgcat::dns_cache::{CachedResolverConfig, CachedResolver}; + /// + /// # tokio_test::block_on(async { + /// let config = CachedResolverConfig::default(); + /// let resolver = CachedResolver::new(config, None).await.unwrap(); + /// # }) + /// ``` + /// + pub async fn new( + config: CachedResolverConfig, + data: Option>, + ) -> Result, io::Error> { + // Construct a new Resolver with default configuration options + let resolver = Some(TokioAsyncResolver::tokio_from_system_conf()?); + + let data = if let Some(hash) = data { + Some(RwLock::new(hash)) + } else { + Some(RwLock::new(HashMap::new())) + }; + + let instance = Arc::new(Self { + config, + resolver, + data, + refresh_loop: RwLock::new(None), + }); + + if instance.enabled() { + info!("Scheduling DNS refresh loop"); + let refresh_loop = tokio::task::spawn({ + let instance = instance.clone(); + async move { + instance.refresh_dns_entries_loop().await; + } + }); + *(instance.refresh_loop.write().unwrap()) = Some(refresh_loop); + } + + Ok(instance) + } + + pub fn enabled(&self) -> bool { + self.config.enabled + } + + // Schedules the refresher + async fn refresh_dns_entries_loop(&self) { + let resolver = TokioAsyncResolver::tokio_from_system_conf().unwrap(); + let interval = Duration::from_secs(self.config.dns_max_ttl); + loop { + debug!("Begin refreshing cached DNS addresses."); + // To minimize the time we hold the lock, we first create + // an array with keys. + let mut hostnames: Vec = Vec::new(); + { + if let Some(ref data) = self.data { + for hostname in data.read().unwrap().keys() { + hostnames.push(hostname.clone()); + } + } + } + + for hostname in hostnames.iter() { + let addrset = self + .fetch_from_cache(hostname.as_str()) + .expect("Could not obtain expected address from cache, this should not happen"); + + match resolver.lookup_ip(hostname).await { + Ok(lookup_ip) => { + let new_addrset = AddrSet::from(lookup_ip); + debug!( + "Obtained address for host ({}) -> ({:?})", + hostname, new_addrset + ); + + if addrset != new_addrset { + debug!( + "Addr changed from {:?} to {:?} updating cache.", + addrset, new_addrset + ); + self.store_in_cache(hostname, new_addrset); + } + } + Err(err) => { + error!( + "There was an error trying to resolv {}: ({}).", + hostname, err + ); + } + } + } + debug!("Finished refreshing cached DNS addresses."); + sleep(interval).await; + } + } + + /// Returns a `AddrSet` given the specified hostname. + /// + /// This method first tries to fetch the value from the cache, if it misses + /// then it is resolved and stored in the cache. TTL from records is ignored. + /// + /// # Arguments + /// + /// * `host` - A string slice referencing the hostname to be resolved. + /// + /// # Example: + /// + /// ``` + /// use pgcat::dns_cache::{CachedResolverConfig, CachedResolver}; + /// + /// # tokio_test::block_on(async { + /// let config = CachedResolverConfig::default(); + /// let resolver = CachedResolver::new(config, None).await.unwrap(); + /// let response = resolver.lookup_ip("www.google.com."); + /// # }) + /// ``` + /// + pub async fn lookup_ip(&self, host: &str) -> ResolveResult { + debug!("Lookup up {} in cache", host); + match self.fetch_from_cache(host) { + Some(addr_set) => { + debug!("Cache hit!"); + Ok(addr_set) + } + None => { + debug!("Not found, executing a dns query!"); + if let Some(ref resolver) = self.resolver { + let addr_set = AddrSet::from(resolver.lookup_ip(host).await?); + debug!("Obtained: {:?}", addr_set); + self.store_in_cache(host, addr_set.clone()); + Ok(addr_set) + } else { + Err(ResolveError::from("No resolver available")) + } + } + } + } + + // + // Returns true if the stored host resolution differs from the AddrSet passed. + pub fn has_changed(&self, host: &str, addr_set: &AddrSet) -> bool { + if let Some(fetched_addr_set) = self.fetch_from_cache(host) { + return fetched_addr_set != *addr_set; + } + false + } + + // Fetches an AddrSet from the inner cache adquiring the read lock. + fn fetch_from_cache(&self, key: &str) -> Option { + if let Some(ref hash) = self.data { + if let Some(addr_set) = hash.read().unwrap().get(key) { + return Some(addr_set.clone()); + } + } + None + } + + // Sets up the global CACHED_RESOLVER static variable so we can globally use DNS + // cache. + pub async fn from_config() -> Result<(), Error> { + let cached_resolver = CACHED_RESOLVER.load(); + let desired_config = CachedResolverConfig::from(get_config()); + + if cached_resolver.config != desired_config { + if let Some(ref refresh_loop) = *(cached_resolver.refresh_loop.write().unwrap()) { + warn!("Killing Dnscache refresh loop as its configuration is being reloaded"); + refresh_loop.abort() + } + let new_resolver = if let Some(ref data) = cached_resolver.data { + let data = Some(data.read().unwrap().clone()); + CachedResolver::new(desired_config, data).await + } else { + CachedResolver::new(desired_config, None).await + }; + + match new_resolver { + Ok(ok) => { + CACHED_RESOLVER.store(ok); + Ok(()) + } + Err(err) => { + let message = format!("Error setting up cached_resolver. Error: {:?}, will continue without this feature.", err); + Err(Error::DNSCachedError(message)) + } + } + } else { + Ok(()) + } + } + + // Stores the AddrSet in cache adquiring the write lock. + fn store_in_cache(&self, host: &str, addr_set: AddrSet) { + if let Some(ref data) = self.data { + data.write().unwrap().insert(host.to_string(), addr_set); + } else { + error!("Could not insert, Hash not initialized"); + } + } +} +#[cfg(test)] +mod tests { + use super::*; + use trust_dns_resolver::error::ResolveError; + + #[tokio::test] + async fn new() { + let config = CachedResolverConfig { + dns_max_ttl: 10, + enabled: true, + }; + let resolver = CachedResolver::new(config, None).await; + assert!(resolver.is_ok()); + } + + #[tokio::test] + async fn lookup_ip() { + let config = CachedResolverConfig { + dns_max_ttl: 10, + enabled: true, + }; + let resolver = CachedResolver::new(config, None).await.unwrap(); + let response = resolver.lookup_ip("www.google.com.").await; + assert!(response.is_ok()); + } + + #[tokio::test] + async fn has_changed() { + let config = CachedResolverConfig { + dns_max_ttl: 10, + enabled: true, + }; + let resolver = CachedResolver::new(config, None).await.unwrap(); + let hostname = "www.google.com."; + let response = resolver.lookup_ip(hostname).await; + let addr_set = response.unwrap(); + assert!(!resolver.has_changed(hostname, &addr_set)); + } + + #[tokio::test] + async fn unknown_host() { + let config = CachedResolverConfig { + dns_max_ttl: 10, + enabled: true, + }; + let resolver = CachedResolver::new(config, None).await.unwrap(); + let hostname = "www.idontexists."; + let response = resolver.lookup_ip(hostname).await; + assert!(matches!(response, Err(ResolveError { .. }))); + } + + #[tokio::test] + async fn incorrect_address() { + let config = CachedResolverConfig { + dns_max_ttl: 10, + enabled: true, + }; + let resolver = CachedResolver::new(config, None).await.unwrap(); + let hostname = "w ww.idontexists."; + let response = resolver.lookup_ip(hostname).await; + assert!(matches!(response, Err(ResolveError { .. }))); + assert!(!resolver.has_changed(hostname, &AddrSet::new())); + } + + #[tokio::test] + // Ok, this test is based on the fact that google does DNS RR + // and does not responds with every available ip everytime, so + // if I cache here, it will miss after one cache iteration or two. + async fn thread() { + let config = CachedResolverConfig { + dns_max_ttl: 10, + enabled: true, + }; + let resolver = CachedResolver::new(config, None).await.unwrap(); + let hostname = "www.google.com."; + let response = resolver.lookup_ip(hostname).await; + let addr_set = response.unwrap(); + assert!(!resolver.has_changed(hostname, &addr_set)); + let resolver_for_refresher = resolver.clone(); + let _thread_handle = tokio::task::spawn(async move { + resolver_for_refresher.refresh_dns_entries_loop().await; + }); + assert!(!resolver.has_changed(hostname, &addr_set)); + } +} diff --git a/src/errors.rs b/src/errors.rs index 0930ab8bd..fb70c0426 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -19,6 +19,7 @@ pub enum Error { ClientError(String), TlsError, StatementTimeout, + DNSCachedError(String), ShuttingDown, ParseBytesError(String), AuthError(String), diff --git a/src/lib.rs b/src/lib.rs index 2645cd427..3a58bb380 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ pub mod auth_passthrough; pub mod config; pub mod constants; +pub mod dns_cache; pub mod errors; pub mod messages; pub mod mirrors; diff --git a/src/main.rs b/src/main.rs index b3265ed87..dc48dd583 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,6 +36,7 @@ extern crate sqlparser; extern crate tokio; extern crate tokio_rustls; extern crate toml; +extern crate trust_dns_resolver; #[cfg(not(target_env = "msvc"))] use jemallocator::Jemalloc; @@ -65,6 +66,7 @@ mod auth_passthrough; mod client; mod config; mod constants; +mod dns_cache; mod errors; mod messages; mod mirrors; @@ -166,8 +168,14 @@ fn main() -> Result<(), Box> { // Statistics reporting. REPORTER.store(Arc::new(Reporter::default())); - // Connection pool that allows to query all shards and replicas. - match ConnectionPool::from_config(client_server_map.clone()).await { + // Starts (if enabled) dns cache before pools initialization + match dns_cache::CachedResolver::from_config().await { + Ok(_) => (), + Err(err) => error!("DNS cache initialization error: {:?}", err), + }; + + // Connection pool that allows to query all shards and replicas. + match ConnectionPool::from_config(client_server_map.clone()).await { Ok(_) => (), Err(err) => { error!("Pool error: {:?}", err); diff --git a/src/server.rs b/src/server.rs index 5bcd5fb99..ff5ab20b5 100644 --- a/src/server.rs +++ b/src/server.rs @@ -7,6 +7,7 @@ use parking_lot::{Mutex, RwLock}; use postgres_protocol::message; use std::collections::HashMap; use std::io::Read; +use std::net::IpAddr; use std::sync::Arc; use std::time::SystemTime; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, BufStream}; @@ -16,6 +17,7 @@ use tokio_rustls::{client::TlsStream, TlsConnector}; use crate::config::{get_config, Address, User}; use crate::constants::*; +use crate::dns_cache::{AddrSet, CACHED_RESOLVER}; use crate::errors::{Error, ServerIdentifier}; use crate::messages::*; use crate::mirrors::MirroringManager; @@ -148,6 +150,9 @@ pub struct Server { last_activity: SystemTime, mirror_manager: Option, + + // Associated addresses used + addr_set: Option, } impl Server { @@ -161,6 +166,24 @@ impl Server { stats: Arc, auth_hash: Arc>>, ) -> Result { + let cached_resolver = CACHED_RESOLVER.load(); + let mut addr_set: Option = None; + + // If we are caching addresses and hostname is not an IP + if cached_resolver.enabled() && address.host.parse::().is_err() { + debug!("Resolving {}", &address.host); + addr_set = match cached_resolver.lookup_ip(&address.host).await { + Ok(ok) => { + debug!("Obtained: {:?}", ok); + Some(ok) + } + Err(err) => { + warn!("Error trying to resolve {}, ({:?})", &address.host, err); + None + } + } + }; + let mut stream = match TcpStream::connect(&format!("{}:{}", &address.host, address.port)).await { Ok(stream) => stream, @@ -609,6 +632,7 @@ impl Server { bad: false, needs_cleanup: false, client_server_map, + addr_set, connected_at: chrono::offset::Utc::now().naive_utc(), stats, application_name: String::new(), @@ -849,7 +873,23 @@ impl Server { /// Server & client are out of sync, we must discard this connection. /// This happens with clients that misbehave. pub fn is_bad(&self) -> bool { - self.bad + if self.bad { + return self.bad; + }; + let cached_resolver = CACHED_RESOLVER.load(); + if cached_resolver.enabled() { + if let Some(addr_set) = &self.addr_set { + if cached_resolver.has_changed(self.address.host.as_str(), addr_set) { + warn!( + "DNS changed for {}, it was {:?}. Dropping server connection.", + self.address.host.as_str(), + addr_set + ); + return true; + } + } + } + false } /// Get server startup information to forward it to the client. From 06ccecd02728e9c6d1d87d7bf6be8831473c89a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 May 2023 07:32:44 -0700 Subject: [PATCH 393/551] chore(deps): bump rustls from 0.21.0 to 0.21.1 (#419) Bumps [rustls](https://github.com/rustls/rustls) from 0.21.0 to 0.21.1. - [Release notes](https://github.com/rustls/rustls/releases) - [Changelog](https://github.com/rustls/rustls/blob/main/RELEASE_NOTES.md) - [Commits](https://github.com/rustls/rustls/compare/v/0.21.0...v/0.21.1) --- updated-dependencies: - dependency-name: rustls dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7641e9c29..dad2e6c55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1145,9 +1145,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07180898a28ed6a7f7ba2311594308f595e3dd2e3c3812fa0a80a47b45f17e5d" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" dependencies = [ "log", "ring", From 90789bb2e014c3f0af9cdece44d53e242a0eeaa4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 3 May 2023 09:13:05 -0700 Subject: [PATCH 394/551] Plugins! (#420) * Some queries * Plugins!! * cleanup * actual names * the actual plugins * comment * fix tests * Tests * unused errors * Increase reaper rate to actually enforce settings * ok --- Cargo.lock | 35 +++- Cargo.toml | 7 +- pgcat.toml | 3 + src/admin.rs | 2 +- src/client.rs | 91 +++++++++- src/config.rs | 4 + src/errors.rs | 4 +- src/lib.rs | 5 + src/main.rs | 54 ++---- src/messages.rs | 35 ++++ src/plugins/intercept.rs | 278 +++++++++++++++++++++++++++++ src/plugins/mod.rs | 40 +++++ src/plugins/table_access.rs | 50 ++++++ src/pool.rs | 24 +++ src/query_router.rs | 221 +++++++++++++++++------ tests/ruby/helpers/pgcat_helper.rb | 1 + tests/ruby/plugins_spec.rb | 14 ++ 17 files changed, 772 insertions(+), 96 deletions(-) create mode 100644 src/plugins/intercept.rs create mode 100644 src/plugins/mod.rs create mode 100644 src/plugins/table_access.rs create mode 100644 tests/ruby/plugins_spec.rb diff --git a/Cargo.lock b/Cargo.lock index dad2e6c55..6724a92cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -882,7 +882,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pgcat" -version = "1.0.1" +version = "1.0.2-alpha1" dependencies = [ "arc-swap", "async-trait", @@ -913,6 +913,7 @@ dependencies = [ "rustls-pemfile", "serde", "serde_derive", + "serde_json", "sha-1", "sha2", "socket2", @@ -1174,6 +1175,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1201,6 +1208,9 @@ name = "serde" version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" @@ -1213,6 +1223,17 @@ dependencies = [ "syn 2.0.9", ] +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.1" @@ -1297,6 +1318,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "355dc4d4b6207ca8a3434fc587db0a8016130a574dbcdbfb93d7f7b5bc5b211a" dependencies = [ "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55fe75cb4a364c7f7ae06c7dbbc8d84bddd85d6cdf9975963c3935bc1991761e" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 436c3dd54..805498215 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.0.1" +version = "1.0.2-alpha1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -14,12 +14,12 @@ rand = "0.8" chrono = "0.4" sha-1 = "0.10" toml = "0.7" -serde = "1" +serde = { version = "1", features = ["derive"] } serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = "0.33.0" +sqlparser = {version = "0.33", features = ["visitor"] } log = "0.4" arc-swap = "1" env_logger = "0.10" @@ -44,6 +44,7 @@ webpki-roots = "0.23" rustls = { version = "0.21", features = ["dangerous_configuration"] } trust-dns-resolver = "0.22.0" tokio-test = "0.4.2" +serde_json = "1" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/pgcat.toml b/pgcat.toml index c844ce1fc..dfb578226 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -77,6 +77,9 @@ admin_username = "admin_user" # Password to access the virtual administrative database admin_password = "admin_pass" +# Plugins!! +# query_router_plugins = ["pg_table_access", "intercept"] + # pool configs are structured as pool. # the pool_name is what clients use as database name when connecting. # For a pool named `sharded_db`, clients access that pool using connection string like diff --git a/src/admin.rs b/src/admin.rs index 03af755c8..ceba20c86 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -12,9 +12,9 @@ use tokio::time::Instant; use crate::config::{get_config, reload_config, VERSION}; use crate::errors::Error; use crate::messages::*; +use crate::pool::ClientServerMap; use crate::pool::{get_all_pools, get_pool}; use crate::stats::{get_client_stats, get_pool_stats, get_server_stats, ClientState, ServerState}; -use crate::ClientServerMap; pub fn generate_server_info_for_admin() -> BytesMut { let mut server_info = BytesMut::new(); diff --git a/src/client.rs b/src/client.rs index efde75544..331a0da40 100644 --- a/src/client.rs +++ b/src/client.rs @@ -16,6 +16,7 @@ use crate::auth_passthrough::refetch_auth_hash; use crate::config::{get_config, get_idle_client_in_transaction_timeout, Address, PoolMode}; use crate::constants::*; use crate::messages::*; +use crate::plugins::PluginOutput; use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; @@ -765,6 +766,9 @@ where self.stats.register(self.stats.clone()); + // Result returned by one of the plugins. + let mut plugin_output = None; + // Our custom protocol loop. // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocol. @@ -815,7 +819,25 @@ where 'Q' => { if query_router.query_parser_enabled() { - query_router.infer(&message); + if let Ok(ast) = QueryRouter::parse(&message) { + let plugin_result = query_router.execute_plugins(&ast).await; + + match plugin_result { + Ok(PluginOutput::Deny(error)) => { + error_response(&mut self.write, &error).await?; + continue; + } + + Ok(PluginOutput::Intercept(result)) => { + write_all(&mut self.write, result).await?; + continue; + } + + _ => (), + }; + + let _ = query_router.infer(&ast); + } } } @@ -823,7 +845,13 @@ where self.buffer.put(&message[..]); if query_router.query_parser_enabled() { - query_router.infer(&message); + if let Ok(ast) = QueryRouter::parse(&message) { + if let Ok(output) = query_router.execute_plugins(&ast).await { + plugin_output = Some(output); + } + + let _ = query_router.infer(&ast); + } } continue; @@ -857,6 +885,18 @@ where continue; } + // Check on plugin results. + match plugin_output { + Some(PluginOutput::Deny(error)) => { + self.buffer.clear(); + error_response(&mut self.write, &error).await?; + plugin_output = None; + continue; + } + + _ => (), + }; + // Get a pool instance referenced by the most up-to-date // pointer. This ensures we always read the latest config // when starting a query. @@ -1085,6 +1125,27 @@ where match code { // Query 'Q' => { + if query_router.query_parser_enabled() { + if let Ok(ast) = QueryRouter::parse(&message) { + let plugin_result = query_router.execute_plugins(&ast).await; + + match plugin_result { + Ok(PluginOutput::Deny(error)) => { + error_response(&mut self.write, &error).await?; + continue; + } + + Ok(PluginOutput::Intercept(result)) => { + write_all(&mut self.write, result).await?; + continue; + } + + _ => (), + }; + + let _ = query_router.infer(&ast); + } + } debug!("Sending query to server"); self.send_and_receive_loop( @@ -1124,6 +1185,14 @@ where // Parse // The query with placeholders is here, e.g. `SELECT * FROM users WHERE email = $1 AND active = $2`. 'P' => { + if query_router.query_parser_enabled() { + if let Ok(ast) = QueryRouter::parse(&message) { + if let Ok(output) = query_router.execute_plugins(&ast).await { + plugin_output = Some(output); + } + } + } + self.buffer.put(&message[..]); } @@ -1155,6 +1224,24 @@ where 'S' => { debug!("Sending query to server"); + match plugin_output { + Some(PluginOutput::Deny(error)) => { + error_response(&mut self.write, &error).await?; + plugin_output = None; + self.buffer.clear(); + continue; + } + + Some(PluginOutput::Intercept(result)) => { + write_all(&mut self.write, result).await?; + plugin_output = None; + self.buffer.clear(); + continue; + } + + _ => (), + }; + self.buffer.put(&message[..]); let first_message_code = (*self.buffer.get(0).unwrap_or(&0)) as char; diff --git a/src/config.rs b/src/config.rs index fd7d3912c..aa98421cc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -298,9 +298,12 @@ pub struct General { pub admin_username: String, pub admin_password: String, + // Support for auth query pub auth_query: Option, pub auth_query_user: Option, pub auth_query_password: Option, + + pub query_router_plugins: Option>, } impl General { @@ -401,6 +404,7 @@ impl Default for General { auth_query_user: None, auth_query_password: None, server_lifetime: 1000 * 3600 * 24, // 24 hours, + query_router_plugins: None, } } } diff --git a/src/errors.rs b/src/errors.rs index fb70c0426..b1796eee2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,7 +1,7 @@ //! Errors. /// Various errors. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub enum Error { SocketError(String), ClientSocketError(String, ClientIdentifier), @@ -24,6 +24,8 @@ pub enum Error { ParseBytesError(String), AuthError(String), AuthPassthroughError(String), + UnsupportedStatement, + QueryRouterParserError(String), } #[derive(Clone, PartialEq, Debug)] diff --git a/src/lib.rs b/src/lib.rs index 3a58bb380..db6167db0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,6 @@ +pub mod admin; pub mod auth_passthrough; +pub mod client; pub mod config; pub mod constants; pub mod dns_cache; @@ -6,7 +8,10 @@ pub mod errors; pub mod messages; pub mod mirrors; pub mod multi_logger; +pub mod plugins; pub mod pool; +pub mod prometheus; +pub mod query_router; pub mod scram; pub mod server; pub mod sharding; diff --git a/src/main.rs b/src/main.rs index dc48dd583..6af4db95b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,37 +61,19 @@ use std::str::FromStr; use std::sync::Arc; use tokio::sync::broadcast; -mod admin; -mod auth_passthrough; -mod client; -mod config; -mod constants; -mod dns_cache; -mod errors; -mod messages; -mod mirrors; -mod multi_logger; -mod pool; -mod prometheus; -mod query_router; -mod scram; -mod server; -mod sharding; -mod stats; -mod tls; - -use crate::config::{get_config, reload_config, VERSION}; -use crate::messages::configure_socket; -use crate::pool::{ClientServerMap, ConnectionPool}; -use crate::prometheus::start_metric_server; -use crate::stats::{Collector, Reporter, REPORTER}; +use pgcat::config::{get_config, reload_config, VERSION}; +use pgcat::dns_cache; +use pgcat::messages::configure_socket; +use pgcat::pool::{ClientServerMap, ConnectionPool}; +use pgcat::prometheus::start_metric_server; +use pgcat::stats::{Collector, Reporter, REPORTER}; fn main() -> Result<(), Box> { - multi_logger::MultiLogger::init().unwrap(); + pgcat::multi_logger::MultiLogger::init().unwrap(); info!("Welcome to PgCat! Meow. (Version {})", VERSION); - if !query_router::QueryRouter::setup() { + if !pgcat::query_router::QueryRouter::setup() { error!("Could not setup query router"); std::process::exit(exitcode::CONFIG); } @@ -109,7 +91,7 @@ fn main() -> Result<(), Box> { let runtime = Builder::new_multi_thread().worker_threads(1).build()?; runtime.block_on(async { - match config::parse(&config_file).await { + match pgcat::config::parse(&config_file).await { Ok(_) => (), Err(err) => { error!("Config parse error: {:?}", err); @@ -168,14 +150,14 @@ fn main() -> Result<(), Box> { // Statistics reporting. REPORTER.store(Arc::new(Reporter::default())); - // Starts (if enabled) dns cache before pools initialization - match dns_cache::CachedResolver::from_config().await { - Ok(_) => (), - Err(err) => error!("DNS cache initialization error: {:?}", err), - }; + // Starts (if enabled) dns cache before pools initialization + match dns_cache::CachedResolver::from_config().await { + Ok(_) => (), + Err(err) => error!("DNS cache initialization error: {:?}", err), + }; - // Connection pool that allows to query all shards and replicas. - match ConnectionPool::from_config(client_server_map.clone()).await { + // Connection pool that allows to query all shards and replicas. + match ConnectionPool::from_config(client_server_map.clone()).await { Ok(_) => (), Err(err) => { error!("Pool error: {:?}", err); @@ -303,7 +285,7 @@ fn main() -> Result<(), Box> { tokio::task::spawn(async move { let start = chrono::offset::Utc::now().naive_utc(); - match client::client_entrypoint( + match pgcat::client::client_entrypoint( socket, client_server_map, shutdown_rx, @@ -334,7 +316,7 @@ fn main() -> Result<(), Box> { Err(err) => { match err { - errors::Error::ClientBadStartup => debug!("Client disconnected with error {:?}", err), + pgcat::errors::Error::ClientBadStartup => debug!("Client disconnected with error {:?}", err), _ => warn!("Client disconnected with error {:?}", err), } diff --git a/src/messages.rs b/src/messages.rs index 0e980fe6b..ee4886df2 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -20,6 +20,10 @@ pub enum DataType { Text, Int4, Numeric, + Bool, + Oid, + AnyArray, + Any, } impl From<&DataType> for i32 { @@ -28,6 +32,10 @@ impl From<&DataType> for i32 { DataType::Text => 25, DataType::Int4 => 23, DataType::Numeric => 1700, + DataType::Bool => 16, + DataType::Oid => 26, + DataType::AnyArray => 2277, + DataType::Any => 2276, } } } @@ -443,6 +451,10 @@ pub fn row_description(columns: &Vec<(&str, DataType)>) -> BytesMut { DataType::Text => -1, DataType::Int4 => 4, DataType::Numeric => -1, + DataType::Bool => 1, + DataType::Oid => 4, + DataType::AnyArray => -1, + DataType::Any => -1, }; row_desc.put_i16(type_size); @@ -481,6 +493,29 @@ pub fn data_row(row: &Vec) -> BytesMut { res } +pub fn data_row_nullable(row: &Vec>) -> BytesMut { + let mut res = BytesMut::new(); + let mut data_row = BytesMut::new(); + + data_row.put_i16(row.len() as i16); + + for column in row { + if let Some(column) = column { + let column = column.as_bytes(); + data_row.put_i32(column.len() as i32); + data_row.put_slice(column); + } else { + data_row.put_i32(-1 as i32); + } + } + + res.put_u8(b'D'); + res.put_i32(data_row.len() as i32 + 4); + res.put(data_row); + + res +} + /// Create a CommandComplete message. pub fn command_complete(command: &str) -> BytesMut { let cmd = BytesMut::from(format!("{}\0", command).as_bytes()); diff --git a/src/plugins/intercept.rs b/src/plugins/intercept.rs new file mode 100644 index 000000000..6e250dcaa --- /dev/null +++ b/src/plugins/intercept.rs @@ -0,0 +1,278 @@ +//! The intercept plugin. +//! +//! It intercepts queries and returns fake results. + +use arc_swap::ArcSwap; +use async_trait::async_trait; +use bytes::{BufMut, BytesMut}; +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use sqlparser::ast::Statement; +use std::collections::HashMap; + +use log::debug; +use std::sync::Arc; + +use crate::{ + errors::Error, + messages::{command_complete, data_row_nullable, row_description, DataType}, + plugins::{Plugin, PluginOutput}, + pool::{PoolIdentifier, PoolMap}, + query_router::QueryRouter, +}; + +pub static CONFIG: Lazy>> = + Lazy::new(|| ArcSwap::from_pointee(HashMap::new())); + +/// Configure the intercept plugin. +pub fn configure(pools: &PoolMap) { + let mut config = HashMap::new(); + for (identifier, _) in pools.iter() { + // TODO: make this configurable from a text config. + let value = fool_datagrip(&identifier.db, &identifier.user); + config.insert(identifier.clone(), value); + } + + CONFIG.store(Arc::new(config)); +} + +// TODO: use these structs for deserialization +#[derive(Serialize, Deserialize)] +pub struct Rule { + query: String, + schema: Vec, + result: Vec>, +} + +#[derive(Serialize, Deserialize)] +pub struct Column { + name: String, + data_type: String, +} + +/// The intercept plugin. +pub struct Intercept; + +#[async_trait] +impl Plugin for Intercept { + async fn run( + &mut self, + query_router: &QueryRouter, + ast: &Vec, + ) -> Result { + if ast.is_empty() { + return Ok(PluginOutput::Allow); + } + + let mut result = BytesMut::new(); + let query_map = match CONFIG.load().get(&PoolIdentifier::new( + &query_router.pool_settings().db, + &query_router.pool_settings().user.username, + )) { + Some(query_map) => query_map.clone(), + None => return Ok(PluginOutput::Allow), + }; + + for q in ast { + // Normalization + let q = q.to_string().to_ascii_lowercase(); + + for target in query_map.as_array().unwrap().iter() { + if target["query"].as_str().unwrap() == q { + debug!("Query matched: {}", q); + + let rd = target["schema"] + .as_array() + .unwrap() + .iter() + .map(|row| { + let row = row.as_object().unwrap(); + ( + row["name"].as_str().unwrap(), + match row["data_type"].as_str().unwrap() { + "text" => DataType::Text, + "anyarray" => DataType::AnyArray, + "oid" => DataType::Oid, + "bool" => DataType::Bool, + "int4" => DataType::Int4, + _ => DataType::Any, + }, + ) + }) + .collect::>(); + + result.put(row_description(&rd)); + + target["result"].as_array().unwrap().iter().for_each(|row| { + let row = row + .as_array() + .unwrap() + .iter() + .map(|s| { + let s = s.as_str().unwrap().to_string(); + + if s == "" { + None + } else { + Some(s) + } + }) + .collect::>>(); + result.put(data_row_nullable(&row)); + }); + + result.put(command_complete("SELECT")); + } + } + } + + if !result.is_empty() { + result.put_u8(b'Z'); + result.put_i32(5); + result.put_u8(b'I'); + + return Ok(PluginOutput::Intercept(result)); + } else { + Ok(PluginOutput::Allow) + } + } +} + +/// Make IntelliJ SQL plugin believe it's talking to an actual database +/// instead of PgCat. +fn fool_datagrip(database: &str, user: &str) -> Value { + json!([ + { + "query": "select current_database() as a, current_schemas(false) as b", + "schema": [ + { + "name": "a", + "data_type": "text", + }, + { + "name": "b", + "data_type": "anyarray", + }, + ], + + "result": [ + [database, "{public}"], + ], + }, + { + "query": "select current_database(), current_schema(), current_user", + "schema": [ + { + "name": "current_database", + "data_type": "text", + }, + { + "name": "current_schema", + "data_type": "text", + }, + { + "name": "current_user", + "data_type": "text", + } + ], + + "result": [ + ["sharded_db", "public", "sharding_user"], + ], + }, + { + "query": "select cast(n.oid as bigint) as id, datname as name, d.description, datistemplate as is_template, datallowconn as allow_connections, pg_catalog.pg_get_userbyid(n.datdba) as \"owner\" from pg_catalog.pg_database as n left join pg_catalog.pg_shdescription as d on n.oid = d.objoid order by case when datname = pg_catalog.current_database() then -cast(1 as bigint) else cast(n.oid as bigint) end", + "schema": [ + { + "name": "id", + "data_type": "oid", + }, + { + "name": "name", + "data_type": "text", + }, + { + "name": "description", + "data_type": "text", + }, + { + "name": "is_template", + "data_type": "bool", + }, + { + "name": "allow_connections", + "data_type": "bool", + }, + { + "name": "owner", + "data_type": "text", + } + ], + "result": [ + ["16387", database, "", "f", "t", user], + ] + }, + { + "query": "select cast(r.oid as bigint) as role_id, rolname as role_name, rolsuper as is_super, rolinherit as is_inherit, rolcreaterole as can_createrole, rolcreatedb as can_createdb, rolcanlogin as can_login, rolreplication as is_replication, rolconnlimit as conn_limit, rolvaliduntil as valid_until, rolbypassrls as bypass_rls, rolconfig as config, d.description from pg_catalog.pg_roles as r left join pg_catalog.pg_shdescription as d on d.objoid = r.oid", + "schema": [ + { + "name": "role_id", + "data_type": "oid", + }, + { + "name": "role_name", + "data_type": "text", + }, + { + "name": "is_super", + "data_type": "bool", + }, + { + "name": "is_inherit", + "data_type": "bool", + }, + { + "name": "can_createrole", + "data_type": "bool", + }, + { + "name": "can_createdb", + "data_type": "bool", + }, + { + "name": "can_login", + "data_type": "bool", + }, + { + "name": "is_replication", + "data_type": "bool", + }, + { + "name": "conn_limit", + "data_type": "int4", + }, + { + "name": "valid_until", + "data_type": "text", + }, + { + "name": "bypass_rls", + "data_type": "bool", + }, + { + "name": "config", + "data_type": "text", + }, + { + "name": "description", + "data_type": "text", + }, + ], + "result": [ + ["10", "postgres", "f", "t", "f", "f", "t", "f", "-1", "", "f", "", ""], + ["16419", user, "f", "t", "f", "f", "t", "f", "-1", "", "f", "", ""], + ] + } + ]) +} diff --git a/src/plugins/mod.rs b/src/plugins/mod.rs new file mode 100644 index 000000000..92fa70b7f --- /dev/null +++ b/src/plugins/mod.rs @@ -0,0 +1,40 @@ +//! The plugin ecosystem. +//! +//! Currently plugins only grant access or deny access to the database for a particual query. +//! Example use cases: +//! - block known bad queries +//! - block access to system catalogs +//! - block dangerous modifications like `DROP TABLE` +//! - etc +//! + +pub mod intercept; +pub mod table_access; + +use crate::{errors::Error, query_router::QueryRouter}; +use async_trait::async_trait; +use bytes::BytesMut; +use sqlparser::ast::Statement; + +pub use intercept::Intercept; +pub use table_access::TableAccess; + +#[derive(Clone, Debug, PartialEq)] +pub enum PluginOutput { + Allow, + Deny(String), + Overwrite(Vec), + Intercept(BytesMut), +} + +#[async_trait] +pub trait Plugin { + // Custom output is allowed because we want to extend this system + // to rewriting queries some day. So an output of a plugin could be + // a rewritten AST. + async fn run( + &mut self, + query_router: &QueryRouter, + ast: &Vec, + ) -> Result; +} diff --git a/src/plugins/table_access.rs b/src/plugins/table_access.rs new file mode 100644 index 000000000..2e23278a3 --- /dev/null +++ b/src/plugins/table_access.rs @@ -0,0 +1,50 @@ +//! This query router plugin will check if the user can access a particular +//! table as part of their query. If they can't, the query will not be routed. + +use async_trait::async_trait; +use sqlparser::ast::{visit_relations, Statement}; + +use crate::{ + errors::Error, + plugins::{Plugin, PluginOutput}, + query_router::QueryRouter, +}; + +use core::ops::ControlFlow; + +pub struct TableAccess { + pub forbidden_tables: Vec, +} + +#[async_trait] +impl Plugin for TableAccess { + async fn run( + &mut self, + _query_router: &QueryRouter, + ast: &Vec, + ) -> Result { + let mut found = None; + + visit_relations(ast, |relation| { + let relation = relation.to_string(); + let parts = relation.split(".").collect::>(); + let table_name = parts.last().unwrap(); + + if self.forbidden_tables.contains(&table_name.to_string()) { + found = Some(table_name.to_string()); + ControlFlow::<()>::Break(()) + } else { + ControlFlow::<()>::Continue(()) + } + }); + + if let Some(found) = found { + Ok(PluginOutput::Deny(format!( + "permission for table \"{}\" denied", + found + ))) + } else { + Ok(PluginOutput::Allow) + } + } +} diff --git a/src/pool.rs b/src/pool.rs index ee8de446f..b986548a4 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -61,6 +61,8 @@ pub struct PoolIdentifier { pub user: String, } +static POOL_REAPER_RATE: u64 = 30_000; // 30 seconds by default + impl PoolIdentifier { /// Create a new user/pool identifier. pub fn new(db: &str, user: &str) -> PoolIdentifier { @@ -91,6 +93,7 @@ pub struct PoolSettings { // Connecting user. pub user: User, + pub db: String, // Default server role to connect to. pub default_role: Option, @@ -129,6 +132,8 @@ pub struct PoolSettings { pub auth_query: Option, pub auth_query_user: Option, pub auth_query_password: Option, + + pub plugins: Option>, } impl Default for PoolSettings { @@ -138,6 +143,7 @@ impl Default for PoolSettings { load_balancing_mode: LoadBalancingMode::Random, shards: 1, user: User::default(), + db: String::default(), default_role: None, query_parser_enabled: false, primary_reads_enabled: true, @@ -152,6 +158,7 @@ impl Default for PoolSettings { auth_query: None, auth_query_user: None, auth_query_password: None, + plugins: None, } } } @@ -368,12 +375,20 @@ impl ConnectionPool { }, }; + let reaper_rate = *vec![idle_timeout, server_lifetime, POOL_REAPER_RATE] + .iter() + .min() + .unwrap(); + + debug!("Pool reaper rate: {}ms", reaper_rate); + let pool = Pool::builder() .max_size(user.pool_size) .min_idle(user.min_pool_size) .connection_timeout(std::time::Duration::from_millis(connect_timeout)) .idle_timeout(Some(std::time::Duration::from_millis(idle_timeout))) .max_lifetime(Some(std::time::Duration::from_millis(server_lifetime))) + .reaper_rate(std::time::Duration::from_millis(reaper_rate)) .test_on_check_out(false) .build(manager) .await?; @@ -412,6 +427,7 @@ impl ConnectionPool { // shards: pool_config.shards.clone(), shards: shard_ids.len(), user: user.clone(), + db: pool_name.clone(), default_role: match pool_config.default_role.as_str() { "any" => None, "replica" => Some(Role::Replica), @@ -437,6 +453,7 @@ impl ConnectionPool { auth_query: pool_config.auth_query.clone(), auth_query_user: pool_config.auth_query_user.clone(), auth_query_password: pool_config.auth_query_password.clone(), + plugins: config.general.query_router_plugins.clone(), }, validated: Arc::new(AtomicBool::new(false)), paused: Arc::new(AtomicBool::new(false)), @@ -456,6 +473,13 @@ impl ConnectionPool { } } + // Initialize plugins here if required. + if let Some(plugins) = config.general.query_router_plugins { + if plugins.contains(&String::from("intercept")) { + crate::plugins::intercept::configure(&new_pools); + } + } + POOLS.store(Arc::new(new_pools.clone())); Ok(()) } diff --git a/src/query_router.rs b/src/query_router.rs index 5b2ba0c48..93bcd4f28 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -6,13 +6,16 @@ use once_cell::sync::OnceCell; use regex::{Regex, RegexSet}; use sqlparser::ast::Statement::{Query, StartTransaction}; use sqlparser::ast::{ - BinaryOperator, Expr, Ident, JoinConstraint, JoinOperator, SetExpr, TableFactor, Value, + BinaryOperator, Expr, Ident, JoinConstraint, JoinOperator, SetExpr, Statement, TableFactor, + Value, }; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; use crate::config::Role; +use crate::errors::Error; use crate::messages::BytesMutReader; +use crate::plugins::{Intercept, Plugin, PluginOutput, TableAccess}; use crate::pool::PoolSettings; use crate::sharding::Sharder; @@ -129,6 +132,10 @@ impl QueryRouter { self.pool_settings = pool_settings; } + pub fn pool_settings<'a>(&'a self) -> &'a PoolSettings { + &self.pool_settings + } + /// Try to parse a command and execute it. pub fn try_execute_command(&mut self, message_buffer: &BytesMut) -> Option<(Command, String)> { let mut message_cursor = Cursor::new(message_buffer); @@ -324,10 +331,7 @@ impl QueryRouter { Some((command, value)) } - /// Try to infer which server to connect to based on the contents of the query. - pub fn infer(&mut self, message: &BytesMut) -> bool { - debug!("Inferring role"); - + pub fn parse(message: &BytesMut) -> Result, Error> { let mut message_cursor = Cursor::new(message); let code = message_cursor.get_u8() as char; @@ -353,28 +357,29 @@ impl QueryRouter { query } - _ => return false, + _ => return Err(Error::UnsupportedStatement), }; - let ast = match Parser::parse_sql(&PostgreSqlDialect {}, &query) { - Ok(ast) => ast, + match Parser::parse_sql(&PostgreSqlDialect {}, &query) { + Ok(ast) => Ok(ast), Err(err) => { - // SELECT ... FOR UPDATE won't get parsed correctly. debug!("{}: {}", err, query); - self.active_role = Some(Role::Primary); - return false; + Err(Error::QueryRouterParserError(err.to_string())) } - }; + } + } - debug!("AST: {:?}", ast); + /// Try to infer which server to connect to based on the contents of the query. + pub fn infer(&mut self, ast: &Vec) -> Result<(), Error> { + debug!("Inferring role"); if ast.is_empty() { // That's weird, no idea, let's go to primary self.active_role = Some(Role::Primary); - return false; + return Err(Error::QueryRouterParserError("empty query".into())); } - for q in &ast { + for q in ast { match q { // All transactions go to the primary, probably a write. StartTransaction { .. } => { @@ -418,7 +423,7 @@ impl QueryRouter { }; } - true + Ok(()) } /// Parse the shard number from the Bind message @@ -783,6 +788,32 @@ impl QueryRouter { } } + /// Add your plugins here and execute them. + pub async fn execute_plugins(&self, ast: &Vec) -> Result { + if let Some(plugins) = &self.pool_settings.plugins { + if plugins.contains(&String::from("intercept")) { + let mut intercept = Intercept {}; + let result = intercept.run(&self, ast).await; + + if let Ok(PluginOutput::Intercept(output)) = result { + return Ok(PluginOutput::Intercept(output)); + } + } + + if plugins.contains(&String::from("pg_table_access")) { + let mut table_access = TableAccess { + forbidden_tables: vec![String::from("pg_database"), String::from("pg_roles")], + }; + + if let Ok(PluginOutput::Deny(error)) = table_access.run(&self, ast).await { + return Ok(PluginOutput::Deny(error)); + } + } + } + + Ok(PluginOutput::Allow) + } + fn set_sharding_key(&mut self, sharding_key: i64) -> Option { let sharder = Sharder::new( self.pool_settings.shards, @@ -810,11 +841,22 @@ impl QueryRouter { /// Should we attempt to parse queries? pub fn query_parser_enabled(&self) -> bool { let enabled = match self.query_parser_enabled { - None => self.pool_settings.query_parser_enabled, - Some(value) => value, - }; + None => { + debug!( + "Using pool settings, query_parser_enabled: {}", + self.pool_settings.query_parser_enabled + ); + self.pool_settings.query_parser_enabled + } - debug!("Query parser enabled: {}", enabled); + Some(value) => { + debug!( + "Using query parser override, query_parser_enabled: {}", + value + ); + value + } + }; enabled } @@ -862,7 +904,7 @@ mod test { for query in queries { // It's a recognized query - assert!(qr.infer(&query)); + assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Replica)); } } @@ -881,7 +923,7 @@ mod test { for query in queries { // It's a recognized query - assert!(qr.infer(&query)); + assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Primary)); } } @@ -893,7 +935,7 @@ mod test { let query = simple_query("SELECT * FROM items WHERE id = 5"); assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO on")) != None); - assert!(qr.infer(&query)); + assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), None); } @@ -913,7 +955,7 @@ mod test { res.put(prepared_stmt); res.put_i16(0); - assert!(qr.infer(&res)); + assert!(qr.infer(&QueryRouter::parse(&res).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Replica)); } @@ -1077,11 +1119,11 @@ mod test { assert_eq!(qr.role(), None); let query = simple_query("INSERT INTO test_table VALUES (1)"); - assert!(qr.infer(&query)); + assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Primary)); let query = simple_query("SELECT * FROM test_table"); - assert!(qr.infer(&query)); + assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Replica)); assert!(qr.query_parser_enabled()); @@ -1113,6 +1155,8 @@ mod test { auth_query: None, auth_query_password: None, auth_query_user: None, + db: "test".to_string(), + plugins: None, }; let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); @@ -1142,15 +1186,24 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); - assert!(qr.infer(&simple_query("BEGIN; SELECT 1; COMMIT;"))); + assert!(qr + .infer(&QueryRouter::parse(&simple_query("BEGIN; SELECT 1; COMMIT;")).unwrap()) + .is_ok()); assert_eq!(qr.role(), Role::Primary); - assert!(qr.infer(&simple_query("SELECT 1; SELECT 2;"))); + assert!(qr + .infer(&QueryRouter::parse(&simple_query("SELECT 1; SELECT 2;")).unwrap()) + .is_ok()); assert_eq!(qr.role(), Role::Replica); - assert!(qr.infer(&simple_query( - "SELECT 123; INSERT INTO t VALUES (5); SELECT 1;" - ))); + assert!(qr + .infer( + &QueryRouter::parse(&simple_query( + "SELECT 123; INSERT INTO t VALUES (5); SELECT 1;" + )) + .unwrap() + ) + .is_ok()); assert_eq!(qr.role(), Role::Primary); } @@ -1177,6 +1230,8 @@ mod test { auth_query: None, auth_query_password: None, auth_query_user: None, + db: "test".to_string(), + plugins: None, }; let mut qr = QueryRouter::new(); qr.update_pool_settings(pool_settings.clone()); @@ -1208,47 +1263,84 @@ mod test { qr.pool_settings.automatic_sharding_key = Some("data.id".to_string()); qr.pool_settings.shards = 3; - assert!(qr.infer(&simple_query("SELECT * FROM data WHERE id = 5"))); + assert!(qr + .infer(&QueryRouter::parse(&simple_query("SELECT * FROM data WHERE id = 5")).unwrap()) + .is_ok()); assert_eq!(qr.shard(), 2); - assert!(qr.infer(&simple_query( - "SELECT one, two, three FROM public.data WHERE id = 6" - ))); + assert!(qr + .infer( + &QueryRouter::parse(&simple_query( + "SELECT one, two, three FROM public.data WHERE id = 6" + )) + .unwrap() + ) + .is_ok()); assert_eq!(qr.shard(), 0); - assert!(qr.infer(&simple_query( - "SELECT * FROM data + assert!(qr + .infer( + &QueryRouter::parse(&simple_query( + "SELECT * FROM data INNER JOIN t2 ON data.id = 5 AND t2.data_id = data.id WHERE data.id = 5" - ))); + )) + .unwrap() + ) + .is_ok()); assert_eq!(qr.shard(), 2); // Shard did not move because we couldn't determine the sharding key since it could be ambiguous // in the query. - assert!(qr.infer(&simple_query( - "SELECT * FROM t2 INNER JOIN data ON id = 6 AND data.id = t2.data_id" - ))); + assert!(qr + .infer( + &QueryRouter::parse(&simple_query( + "SELECT * FROM t2 INNER JOIN data ON id = 6 AND data.id = t2.data_id" + )) + .unwrap() + ) + .is_ok()); assert_eq!(qr.shard(), 2); - assert!(qr.infer(&simple_query( - r#"SELECT * FROM "public"."data" WHERE "id" = 6"# - ))); + assert!(qr + .infer( + &QueryRouter::parse(&simple_query( + r#"SELECT * FROM "public"."data" WHERE "id" = 6"# + )) + .unwrap() + ) + .is_ok()); assert_eq!(qr.shard(), 0); - assert!(qr.infer(&simple_query( - r#"SELECT * FROM "public"."data" WHERE "data"."id" = 5"# - ))); + assert!(qr + .infer( + &QueryRouter::parse(&simple_query( + r#"SELECT * FROM "public"."data" WHERE "data"."id" = 5"# + )) + .unwrap() + ) + .is_ok()); assert_eq!(qr.shard(), 2); // Super unique sharding key qr.pool_settings.automatic_sharding_key = Some("*.unique_enough_column_name".to_string()); - assert!(qr.infer(&simple_query( - "SELECT * FROM table_x WHERE unique_enough_column_name = 6" - ))); + assert!(qr + .infer( + &QueryRouter::parse(&simple_query( + "SELECT * FROM table_x WHERE unique_enough_column_name = 6" + )) + .unwrap() + ) + .is_ok()); assert_eq!(qr.shard(), 0); - assert!(qr.infer(&simple_query("SELECT * FROM table_y WHERE another_key = 5"))); + assert!(qr + .infer( + &QueryRouter::parse(&simple_query("SELECT * FROM table_y WHERE another_key = 5")) + .unwrap() + ) + .is_ok()); assert_eq!(qr.shard(), 0); } @@ -1272,11 +1364,36 @@ mod test { qr.pool_settings.automatic_sharding_key = Some("data.id".to_string()); qr.pool_settings.shards = 3; - assert!(qr.infer(&simple_query(stmt))); + assert!(qr + .infer(&QueryRouter::parse(&simple_query(stmt)).unwrap()) + .is_ok()); assert_eq!(qr.placeholders.len(), 1); assert!(qr.infer_shard_from_bind(&bind)); assert_eq!(qr.shard(), 2); assert!(qr.placeholders.is_empty()); } + + #[tokio::test] + async fn test_table_access_plugin() { + QueryRouter::setup(); + + let mut qr = QueryRouter::new(); + + let mut pool_settings = PoolSettings::default(); + pool_settings.plugins = Some(vec![String::from("pg_table_access")]); + qr.update_pool_settings(pool_settings); + + let query = simple_query("SELECT * FROM pg_database"); + let ast = QueryRouter::parse(&query).unwrap(); + + let res = qr.execute_plugins(&ast).await; + + assert_eq!( + res, + Ok(PluginOutput::Deny( + "permission for table \"pg_database\" denied".to_string() + )) + ); + } } diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index ad4c32a47..eb0cdaa9f 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -27,6 +27,7 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mod primary2 = PgInstance.new(8432, user["username"], user["password"], "shard2") pgcat_cfg = pgcat.current_config + pgcat_cfg["general"]["query_router_plugins"] = ["intercept"] pgcat_cfg["pools"] = { "#{pool_name}" => { "default_role" => "any", diff --git a/tests/ruby/plugins_spec.rb b/tests/ruby/plugins_spec.rb new file mode 100644 index 000000000..d4e233ab7 --- /dev/null +++ b/tests/ruby/plugins_spec.rb @@ -0,0 +1,14 @@ +require_relative 'spec_helper' + + +describe "Plugins" do + let(:processes) { Helpers::Pgcat.three_shard_setup("sharded_db", 5) } + + context "intercept" do + it "will intercept an intellij query" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + res = conn.exec("select current_database() as a, current_schemas(false) as b") + expect(res.values).to eq([["sharded_db", "{public}"]]) + end + end +end From 1c93b49c799e5a75420d100542c2283e72ddf94e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 10:00:05 -0700 Subject: [PATCH 395/551] chore(deps): bump regex from 1.8.0 to 1.8.1 (#413) Bumps [regex](https://github.com/rust-lang/regex) from 1.8.0 to 1.8.1. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/commits/1.8.1) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6724a92cf..6b2d76f29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1090,9 +1090,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6cf59af1067a3fb53fbe5c88c053764e930f932be1d71d3ffe032cbe147f59" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", @@ -1101,9 +1101,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6868896879ba532248f33598de5181522d8b3d9d724dfd230911e1a7d4822f5" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" [[package]] name = "resolv-conf" From fc0e5a34a434ae3cae3fa1f49ad56c33ae5efce4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 3 May 2023 16:13:45 -0700 Subject: [PATCH 396/551] Actually plugins (#421) * more plugins * clean up * fix tests * fix flakey test --- Cargo.toml | 1 + pgcat.toml | 42 ++++++++++++++++++-- src/config.rs | 63 ++++++++++++++++++++++++++---- src/plugins/intercept.rs | 50 ++++++++++++++---------- src/plugins/mod.rs | 9 +++-- src/plugins/query_logger.rs | 49 +++++++++++++++++++++++ src/plugins/table_access.rs | 29 ++++++++++++-- src/pool.rs | 31 +++++++++++---- src/query_router.rs | 51 +++++++++++++----------- tests/ruby/admin_spec.rb | 8 ++-- tests/ruby/helpers/pgcat_helper.rb | 1 - 11 files changed, 264 insertions(+), 70 deletions(-) create mode 100644 src/plugins/query_logger.rs diff --git a/Cargo.toml b/Cargo.toml index 805498215..1b3c8b1e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,3 +48,4 @@ serde_json = "1" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" + diff --git a/pgcat.toml b/pgcat.toml index dfb578226..ce3663290 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -77,9 +77,6 @@ admin_username = "admin_user" # Password to access the virtual administrative database admin_password = "admin_pass" -# Plugins!! -# query_router_plugins = ["pg_table_access", "intercept"] - # pool configs are structured as pool. # the pool_name is what clients use as database name when connecting. # For a pool named `sharded_db`, clients access that pool using connection string like @@ -157,6 +154,45 @@ connect_timeout = 3000 # Specifies how often (in seconds) cached ip addresses for servers are rechecked (see `dns_cache_enabled`). # dns_max_ttl = 30 +[plugins] + +[plugins.query_logger] +enabled = false + +[plugins.table_access] +enabled = false +tables = [ + "pg_user", + "pg_roles", + "pg_database", +] + +[plugins.intercept] +enabled = true + +[plugins.intercept.queries.0] + +query = "select current_database() as a, current_schemas(false) as b" +schema = [ + ["a", "text"], + ["b", "text"], +] +result = [ + ["${DATABASE}", "{public}"], +] + +[plugins.intercept.queries.1] + +query = "select current_database(), current_schema(), current_user" +schema = [ + ["current_database", "text"], + ["current_schema", "text"], + ["current_user", "text"], +] +result = [ + ["${DATABASE}", "public", "${USER}"], +] + # User configs are structured as pool..users. # This section holds the credentials for users that may connect to this cluster [pools.sharded_db.users.0] diff --git a/src/config.rs b/src/config.rs index aa98421cc..cdf891eeb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -302,8 +302,6 @@ pub struct General { pub auth_query: Option, pub auth_query_user: Option, pub auth_query_password: Option, - - pub query_router_plugins: Option>, } impl General { @@ -404,7 +402,6 @@ impl Default for General { auth_query_user: None, auth_query_password: None, server_lifetime: 1000 * 3600 * 24, // 24 hours, - query_router_plugins: None, } } } @@ -682,6 +679,55 @@ impl Default for Shard { } } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +pub struct Plugins { + pub intercept: Option, + pub table_access: Option, + pub query_logger: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +pub struct Intercept { + pub enabled: bool, + pub queries: BTreeMap, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +pub struct TableAccess { + pub enabled: bool, + pub tables: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +pub struct QueryLogger { + pub enabled: bool, +} + +impl Intercept { + pub fn substitute(&mut self, db: &str, user: &str) { + for (_, query) in self.queries.iter_mut() { + query.substitute(db, user); + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +pub struct Query { + pub query: String, + pub schema: Vec>, + pub result: Vec>, +} + +impl Query { + pub fn substitute(&mut self, db: &str, user: &str) { + for col in self.result.iter_mut() { + for i in 0..col.len() { + col[i] = col[i].replace("${USER}", user).replace("${DATABASE}", db); + } + } + } +} + /// Configuration wrapper. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Config { @@ -700,6 +746,7 @@ pub struct Config { pub path: String, pub general: General, + pub plugins: Option, pub pools: HashMap, } @@ -737,6 +784,7 @@ impl Default for Config { path: Self::default_path(), general: General::default(), pools: HashMap::default(), + plugins: None, } } } @@ -1128,6 +1176,7 @@ pub async fn parse(path: &str) -> Result<(), Error> { pub async fn reload_config(client_server_map: ClientServerMap) -> Result { let old_config = get_config(); + match parse(&old_config.path).await { Ok(()) => (), Err(err) => { @@ -1135,18 +1184,18 @@ pub async fn reload_config(client_server_map: ClientServerMap) -> Result (), Err(err) => error!("DNS cache reinitialization error: {:?}", err), }; - if old_config.pools != new_config.pools { - info!("Pool configuration changed"); + if old_config != new_config { + info!("Config changed, reloading"); ConnectionPool::from_config(client_server_map).await?; Ok(true) - } else if old_config != new_config { - Ok(true) } else { Ok(false) } diff --git a/src/plugins/intercept.rs b/src/plugins/intercept.rs index 6e250dcaa..88d24d0e4 100644 --- a/src/plugins/intercept.rs +++ b/src/plugins/intercept.rs @@ -11,10 +11,11 @@ use serde_json::{json, Value}; use sqlparser::ast::Statement; use std::collections::HashMap; -use log::debug; +use log::{debug, info}; use std::sync::Arc; use crate::{ + config::Intercept as InterceptConfig, errors::Error, messages::{command_complete, data_row_nullable, row_description, DataType}, plugins::{Plugin, PluginOutput}, @@ -22,19 +23,29 @@ use crate::{ query_router::QueryRouter, }; -pub static CONFIG: Lazy>> = +pub static CONFIG: Lazy>> = Lazy::new(|| ArcSwap::from_pointee(HashMap::new())); -/// Configure the intercept plugin. -pub fn configure(pools: &PoolMap) { +/// Check if the interceptor plugin has been enabled. +pub fn enabled() -> bool { + !CONFIG.load().is_empty() +} + +pub fn setup(intercept_config: &InterceptConfig, pools: &PoolMap) { let mut config = HashMap::new(); for (identifier, _) in pools.iter() { - // TODO: make this configurable from a text config. - let value = fool_datagrip(&identifier.db, &identifier.user); - config.insert(identifier.clone(), value); + let mut intercept_config = intercept_config.clone(); + intercept_config.substitute(&identifier.db, &identifier.user); + config.insert(identifier.clone(), intercept_config); } CONFIG.store(Arc::new(config)); + + info!("Intercepting {} queries", intercept_config.queries.len()); +} + +pub fn disable() { + CONFIG.store(Arc::new(HashMap::new())); } // TODO: use these structs for deserialization @@ -78,19 +89,19 @@ impl Plugin for Intercept { // Normalization let q = q.to_string().to_ascii_lowercase(); - for target in query_map.as_array().unwrap().iter() { - if target["query"].as_str().unwrap() == q { - debug!("Query matched: {}", q); + for (_, target) in query_map.queries.iter() { + if target.query.as_str() == q { + debug!("Intercepting query: {}", q); - let rd = target["schema"] - .as_array() - .unwrap() + let rd = target + .schema .iter() .map(|row| { - let row = row.as_object().unwrap(); + let name = &row[0]; + let data_type = &row[1]; ( - row["name"].as_str().unwrap(), - match row["data_type"].as_str().unwrap() { + name.as_str(), + match data_type.as_str() { "text" => DataType::Text, "anyarray" => DataType::AnyArray, "oid" => DataType::Oid, @@ -104,13 +115,11 @@ impl Plugin for Intercept { result.put(row_description(&rd)); - target["result"].as_array().unwrap().iter().for_each(|row| { + target.result.iter().for_each(|row| { let row = row - .as_array() - .unwrap() .iter() .map(|s| { - let s = s.as_str().unwrap().to_string(); + let s = s.as_str().to_string(); if s == "" { None @@ -141,6 +150,7 @@ impl Plugin for Intercept { /// Make IntelliJ SQL plugin believe it's talking to an actual database /// instead of PgCat. +#[allow(dead_code)] fn fool_datagrip(database: &str, user: &str) -> Value { json!([ { diff --git a/src/plugins/mod.rs b/src/plugins/mod.rs index 92fa70b7f..6661ece63 100644 --- a/src/plugins/mod.rs +++ b/src/plugins/mod.rs @@ -9,6 +9,7 @@ //! pub mod intercept; +pub mod query_logger; pub mod table_access; use crate::{errors::Error, query_router::QueryRouter}; @@ -17,6 +18,7 @@ use bytes::BytesMut; use sqlparser::ast::Statement; pub use intercept::Intercept; +pub use query_logger::QueryLogger; pub use table_access::TableAccess; #[derive(Clone, Debug, PartialEq)] @@ -29,12 +31,13 @@ pub enum PluginOutput { #[async_trait] pub trait Plugin { - // Custom output is allowed because we want to extend this system - // to rewriting queries some day. So an output of a plugin could be - // a rewritten AST. + // Run before the query is sent to the server. async fn run( &mut self, query_router: &QueryRouter, ast: &Vec, ) -> Result; + + // TODO: run after the result is returned + // async fn callback(&mut self, query_router: &QueryRouter); } diff --git a/src/plugins/query_logger.rs b/src/plugins/query_logger.rs new file mode 100644 index 000000000..2dfda8bcf --- /dev/null +++ b/src/plugins/query_logger.rs @@ -0,0 +1,49 @@ +//! Log all queries to stdout (or somewhere else, why not). + +use crate::{ + errors::Error, + plugins::{Plugin, PluginOutput}, + query_router::QueryRouter, +}; +use arc_swap::ArcSwap; +use async_trait::async_trait; +use log::info; +use once_cell::sync::Lazy; +use sqlparser::ast::Statement; +use std::sync::Arc; + +static ENABLED: Lazy> = Lazy::new(|| ArcSwap::from_pointee(false)); + +pub struct QueryLogger; + +pub fn setup() { + ENABLED.store(Arc::new(true)); + + info!("Logging queries to stdout"); +} + +pub fn disable() { + ENABLED.store(Arc::new(false)); +} + +pub fn enabled() -> bool { + **ENABLED.load() +} + +#[async_trait] +impl Plugin for QueryLogger { + async fn run( + &mut self, + _query_router: &QueryRouter, + ast: &Vec, + ) -> Result { + let query = ast + .iter() + .map(|q| q.to_string()) + .collect::>() + .join("; "); + info!("{}", query); + + Ok(PluginOutput::Allow) + } +} diff --git a/src/plugins/table_access.rs b/src/plugins/table_access.rs index 2e23278a3..4613a4fb3 100644 --- a/src/plugins/table_access.rs +++ b/src/plugins/table_access.rs @@ -5,17 +5,37 @@ use async_trait::async_trait; use sqlparser::ast::{visit_relations, Statement}; use crate::{ + config::TableAccess as TableAccessConfig, errors::Error, plugins::{Plugin, PluginOutput}, query_router::QueryRouter, }; +use log::{debug, info}; + +use arc_swap::ArcSwap; use core::ops::ControlFlow; +use once_cell::sync::Lazy; +use std::sync::Arc; + +static CONFIG: Lazy>> = Lazy::new(|| ArcSwap::from_pointee(vec![])); + +pub fn setup(config: &TableAccessConfig) { + CONFIG.store(Arc::new(config.tables.clone())); + + info!("Blocking access to {} tables", config.tables.len()); +} -pub struct TableAccess { - pub forbidden_tables: Vec, +pub fn enabled() -> bool { + !CONFIG.load().is_empty() } +pub fn disable() { + CONFIG.store(Arc::new(vec![])); +} + +pub struct TableAccess; + #[async_trait] impl Plugin for TableAccess { async fn run( @@ -24,13 +44,14 @@ impl Plugin for TableAccess { ast: &Vec, ) -> Result { let mut found = None; + let forbidden_tables = CONFIG.load(); visit_relations(ast, |relation| { let relation = relation.to_string(); let parts = relation.split(".").collect::>(); let table_name = parts.last().unwrap(); - if self.forbidden_tables.contains(&table_name.to_string()) { + if forbidden_tables.contains(&table_name.to_string()) { found = Some(table_name.to_string()); ControlFlow::<()>::Break(()) } else { @@ -39,6 +60,8 @@ impl Plugin for TableAccess { }); if let Some(found) = found { + debug!("Blocking access to table \"{}\"", found); + Ok(PluginOutput::Deny(format!( "permission for table \"{}\" denied", found diff --git a/src/pool.rs b/src/pool.rs index b986548a4..2fd380ce6 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -132,8 +132,6 @@ pub struct PoolSettings { pub auth_query: Option, pub auth_query_user: Option, pub auth_query_password: Option, - - pub plugins: Option>, } impl Default for PoolSettings { @@ -158,7 +156,6 @@ impl Default for PoolSettings { auth_query: None, auth_query_user: None, auth_query_password: None, - plugins: None, } } } @@ -453,7 +450,6 @@ impl ConnectionPool { auth_query: pool_config.auth_query.clone(), auth_query_user: pool_config.auth_query_user.clone(), auth_query_password: pool_config.auth_query_password.clone(), - plugins: config.general.query_router_plugins.clone(), }, validated: Arc::new(AtomicBool::new(false)), paused: Arc::new(AtomicBool::new(false)), @@ -473,10 +469,29 @@ impl ConnectionPool { } } - // Initialize plugins here if required. - if let Some(plugins) = config.general.query_router_plugins { - if plugins.contains(&String::from("intercept")) { - crate::plugins::intercept::configure(&new_pools); + if let Some(ref plugins) = config.plugins { + if let Some(ref intercept) = plugins.intercept { + if intercept.enabled { + crate::plugins::intercept::setup(intercept, &new_pools); + } else { + crate::plugins::intercept::disable(); + } + } + + if let Some(ref table_access) = plugins.table_access { + if table_access.enabled { + crate::plugins::table_access::setup(table_access); + } else { + crate::plugins::table_access::disable(); + } + } + + if let Some(ref query_logger) = plugins.query_logger { + if query_logger.enabled { + crate::plugins::query_logger::setup(); + } else { + crate::plugins::query_logger::disable(); + } } } diff --git a/src/query_router.rs b/src/query_router.rs index 93bcd4f28..d995b804f 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -15,7 +15,10 @@ use sqlparser::parser::Parser; use crate::config::Role; use crate::errors::Error; use crate::messages::BytesMutReader; -use crate::plugins::{Intercept, Plugin, PluginOutput, TableAccess}; +use crate::plugins::{ + intercept, query_logger, table_access, Intercept, Plugin, PluginOutput, QueryLogger, + TableAccess, +}; use crate::pool::PoolSettings; use crate::sharding::Sharder; @@ -790,24 +793,26 @@ impl QueryRouter { /// Add your plugins here and execute them. pub async fn execute_plugins(&self, ast: &Vec) -> Result { - if let Some(plugins) = &self.pool_settings.plugins { - if plugins.contains(&String::from("intercept")) { - let mut intercept = Intercept {}; - let result = intercept.run(&self, ast).await; + if query_logger::enabled() { + let mut query_logger = QueryLogger {}; + let _ = query_logger.run(&self, ast).await; + } - if let Ok(PluginOutput::Intercept(output)) = result { - return Ok(PluginOutput::Intercept(output)); - } + if intercept::enabled() { + let mut intercept = Intercept {}; + let result = intercept.run(&self, ast).await; + + if let Ok(PluginOutput::Intercept(output)) = result { + return Ok(PluginOutput::Intercept(output)); } + } - if plugins.contains(&String::from("pg_table_access")) { - let mut table_access = TableAccess { - forbidden_tables: vec![String::from("pg_database"), String::from("pg_roles")], - }; + if table_access::enabled() { + let mut table_access = TableAccess {}; + let result = table_access.run(&self, ast).await; - if let Ok(PluginOutput::Deny(error)) = table_access.run(&self, ast).await { - return Ok(PluginOutput::Deny(error)); - } + if let Ok(PluginOutput::Deny(error)) = result { + return Ok(PluginOutput::Deny(error)); } } @@ -1156,7 +1161,6 @@ mod test { auth_query_password: None, auth_query_user: None, db: "test".to_string(), - plugins: None, }; let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); @@ -1231,7 +1235,6 @@ mod test { auth_query_password: None, auth_query_user: None, db: "test".to_string(), - plugins: None, }; let mut qr = QueryRouter::new(); qr.update_pool_settings(pool_settings.clone()); @@ -1376,13 +1379,17 @@ mod test { #[tokio::test] async fn test_table_access_plugin() { - QueryRouter::setup(); + use crate::config::TableAccess; + let ta = TableAccess { + enabled: true, + tables: vec![String::from("pg_database")], + }; - let mut qr = QueryRouter::new(); + crate::plugins::table_access::setup(&ta); - let mut pool_settings = PoolSettings::default(); - pool_settings.plugins = Some(vec![String::from("pg_table_access")]); - qr.update_pool_settings(pool_settings); + QueryRouter::setup(); + + let qr = QueryRouter::new(); let query = simple_query("SELECT * FROM pg_database"); let ast = QueryRouter::parse(&query).unwrap(); diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index fceb95bf7..ea21630f9 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -71,15 +71,17 @@ context "client connects but issues no queries" do it "only affects cl_idle stats" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + + before_test = admin_conn.async_exec("SHOW POOLS")[0]["sv_idle"] connections = Array.new(20) { PG::connect(pgcat_conn_str) } sleep(1) - admin_conn = PG::connect(processes.pgcat.admin_connection_string) results = admin_conn.async_exec("SHOW POOLS")[0] %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" end expect(results["cl_idle"]).to eq("20") - expect(results["sv_idle"]).to eq("1") + expect(results["sv_idle"]).to eq(before_test) connections.map(&:close) sleep(1.1) @@ -87,7 +89,7 @@ %w[cl_active cl_idle cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" end - expect(results["sv_idle"]).to eq("1") + expect(results["sv_idle"]).to eq(before_test) end end diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index eb0cdaa9f..ad4c32a47 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -27,7 +27,6 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mod primary2 = PgInstance.new(8432, user["username"], user["password"], "shard2") pgcat_cfg = pgcat.current_config - pgcat_cfg["general"]["query_router_plugins"] = ["intercept"] pgcat_cfg["pools"] = { "#{pool_name}" => { "default_role" => "any", From bdd7ed66c530d2e74b746ec4557467070eba37c7 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 3 May 2023 16:47:20 -0700 Subject: [PATCH 397/551] lowercase config query (#422) * lowercase config query * remove debug --- src/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.rs b/src/config.rs index cdf891eeb..561294079 100644 --- a/src/config.rs +++ b/src/config.rs @@ -707,6 +707,7 @@ impl Intercept { pub fn substitute(&mut self, db: &str, user: &str) { for (_, query) in self.queries.iter_mut() { query.substitute(db, user); + query.query = query.query.to_ascii_lowercase(); } } } From 6e309ced6b64bcffd1b8373b6d335303eeae1a61 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 3 May 2023 17:07:23 -0700 Subject: [PATCH 398/551] Optionally validate config on boot (#423) --- src/config.rs | 8 ++++++++ src/pool.rs | 10 ++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/config.rs b/src/config.rs index 561294079..c7fce635e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -298,6 +298,9 @@ pub struct General { pub admin_username: String, pub admin_password: String, + #[serde(default = "General::default_validate_config")] + pub validate_config: bool, + // Support for auth query pub auth_query: Option, pub auth_query_user: Option, @@ -367,6 +370,10 @@ impl General { pub fn default_idle_client_in_transaction_timeout() -> u64 { 0 } + + pub fn default_validate_config() -> bool { + true + } } impl Default for General { @@ -402,6 +409,7 @@ impl Default for General { auth_query_user: None, auth_query_password: None, server_lifetime: 1000 * 3600 * 24, // 24 hours, + validate_config: true, } } } diff --git a/src/pool.rs b/src/pool.rs index 2fd380ce6..27ff86163 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -459,10 +459,12 @@ impl ConnectionPool { // Connect to the servers to make sure pool configuration is valid // before setting it globally. // Do this async and somewhere else, we don't have to wait here. - let mut validate_pool = pool.clone(); - tokio::task::spawn(async move { - let _ = validate_pool.validate().await; - }); + if config.general.validate_config { + let mut validate_pool = pool.clone(); + tokio::task::spawn(async move { + let _ = validate_pool.validate().await; + }); + } // There is one pool per database/user pair. new_pools.insert(PoolIdentifier::new(pool_name, &user.username), pool); From f2ebcc05d84135ce441b7a8436bae270c46a0207 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 5 May 2023 08:27:19 -0700 Subject: [PATCH 399/551] Accurate log messages (#425) --- src/pool.rs | 14 ++++++++++---- src/server.rs | 22 ++++++++++++++++------ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/pool.rs b/src/pool.rs index 27ff86163..4664193c7 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -641,7 +641,10 @@ impl ConnectionPool { { Ok(conn) => conn, Err(err) => { - error!("Banning instance {:?}, error: {:?}", address, err); + error!( + "Connection checkout error for instance {:?}, error: {:?}", + address, err + ); self.ban(address, BanReason::FailedCheckout, Some(client_stats)); address.stats.error(); client_stats.idle(); @@ -717,7 +720,7 @@ impl ConnectionPool { // Health check failed. Err(err) => { error!( - "Banning instance {:?} because of failed health check, {:?}", + "Failed health check on instance {:?}, error: {:?}", address, err ); } @@ -726,7 +729,7 @@ impl ConnectionPool { // Health check timed out. Err(err) => { error!( - "Banning instance {:?} because of health check timeout, {:?}", + "Health check timeout on instance {:?}, error: {:?}", address, err ); } @@ -748,13 +751,16 @@ impl ConnectionPool { return; } + error!("Banning instance {:?}, reason: {:?}", address, reason); + let now = chrono::offset::Utc::now().naive_utc(); let mut guard = self.banlist.write(); - error!("Banning {:?}", address); + if let Some(client_info) = client_info { client_info.ban_error(); address.stats.error(); } + guard[address.shard].insert(address.clone(), (reason, now)); } diff --git a/src/server.rs b/src/server.rs index ff5ab20b5..152f05246 100644 --- a/src/server.rs +++ b/src/server.rs @@ -705,7 +705,10 @@ impl Server { Ok(()) } Err(err) => { - error!("Terminating server because of: {:?}", err); + error!( + "Terminating server {:?} because of: {:?}", + self.address, err + ); self.bad = true; Err(err) } @@ -720,7 +723,10 @@ impl Server { let mut message = match read_message(&mut self.stream).await { Ok(message) => message, Err(err) => { - error!("Terminating server because of: {:?}", err); + error!( + "Terminating server {:?} because of: {:?}", + self.address, err + ); self.bad = true; return Err(err); } @@ -1135,14 +1141,18 @@ impl Drop for Server { _ => debug!("Dirty shutdown"), }; - // Should not matter. - self.bad = true; - let now = chrono::offset::Utc::now().naive_utc(); let duration = now - self.connected_at; + let message = if self.bad { + "Server connection terminated" + } else { + "Server connection closed" + }; + info!( - "Server connection closed {:?}, session duration: {}", + "{} {:?}, session duration: {}", + message, self.address, crate::format_duration(&duration) ); From ec95b7d6a199e5cbd04b30fd5fc68d7d5d32b5a5 Mon Sep 17 00:00:00 2001 From: Andrew Tanner Date: Fri, 5 May 2023 18:31:27 -0400 Subject: [PATCH 400/551] First try with role reset (#427) * First try with role rest * update * extra line * Update src/server.rs * Update tests/ruby/misc_spec.rb --------- --- src/server.rs | 1 + tests/ruby/misc_spec.rb | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/server.rs b/src/server.rs index 152f05246..8f0ece999 100644 --- a/src/server.rs +++ b/src/server.rs @@ -963,6 +963,7 @@ impl Server { if self.needs_cleanup { warn!("Server returned with session state altered, discarding state"); self.query("DISCARD ALL").await?; + self.query("RESET ROLE").await?; self.needs_cleanup = false; } diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index 1a04c5d67..e4d6f6fbb 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -241,6 +241,18 @@ expect(processes.primary.count_query("DISCARD ALL")).to eq(10) end + + it "Resets server roles correctly" do + 10.times do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SET SERVER ROLE to 'primary'") + conn.async_exec("SELECT 1") + conn.async_exec("SET statement_timeout to 5000") + conn.close + end + + expect(processes.primary.count_query("RESET ROLE")).to eq(10) + end end context "transaction mode" do From a56356932a05a6d3d431083b1a7f86f3818f999d Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 8 May 2023 10:06:16 -0700 Subject: [PATCH 401/551] Calculate averages correctly and preserve totals like before (#429) * Reset totals after avg calculation * like it used to be --- Cargo.lock | 16 ++++++++++++++ Cargo.toml | 1 + src/stats/address.rs | 50 ++++++++++++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b2d76f29..9c553d458 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,6 +250,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + [[package]] name = "enum-as-inner" version = "0.5.1" @@ -658,6 +664,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.5" @@ -897,6 +912,7 @@ dependencies = [ "futures", "hmac", "hyper", + "itertools", "jemallocator", "log", "md-5", diff --git a/Cargo.toml b/Cargo.toml index 1b3c8b1e8..5428ab312 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ rustls = { version = "0.21", features = ["dangerous_configuration"] } trust-dns-resolver = "0.22.0" tokio-test = "0.4.2" serde_json = "1" +itertools = "0.10" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/src/stats/address.rs b/src/stats/address.rs index a5759e100..51d6a688e 100644 --- a/src/stats/address.rs +++ b/src/stats/address.rs @@ -1,4 +1,3 @@ -use log::warn; use std::sync::atomic::*; use std::sync::Arc; @@ -13,6 +12,16 @@ pub struct AddressStats { pub total_query_time: Arc, pub total_wait_time: Arc, pub total_errors: Arc, + + pub old_total_xact_count: Arc, + pub old_total_query_count: Arc, + pub old_total_received: Arc, + pub old_total_sent: Arc, + pub old_total_xact_time: Arc, + pub old_total_query_time: Arc, + pub old_total_wait_time: Arc, + pub old_total_errors: Arc, + pub avg_query_count: Arc, pub avg_query_time: Arc, pub avg_recv: Arc, @@ -104,16 +113,16 @@ impl AddressStats { } pub fn update_averages(&self) { - let (totals, averages) = self.fields_iterators(); - for data in totals.iter().zip(averages.iter()) { - let (total, average) = data; - if let Err(err) = average.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |avg| { - let total = total.load(Ordering::Relaxed); - let avg = (total - avg) / (crate::stats::STAT_PERIOD / 1_000); // Avg / second - Some(avg) - }) { - warn!("Could not update averages for addresses stats, {:?}", err); - } + let (totals, averages, old_totals) = self.fields_iterators(); + for data in itertools::izip!(totals, averages, old_totals) { + let (total, average, old_total) = data; + let total = total.load(Ordering::Relaxed); + let old = old_total.load(Ordering::Relaxed); + average.store( + (total - old) / (crate::stats::STAT_PERIOD / 1_000), + Ordering::Relaxed, + ); // Avg / second + old_total.store(total, Ordering::Relaxed); } } @@ -123,27 +132,42 @@ impl AddressStats { } } - fn fields_iterators(&self) -> (Vec>, Vec>) { + fn fields_iterators( + &self, + ) -> ( + Vec>, + Vec>, + Vec>, + ) { let mut totals: Vec> = Vec::new(); let mut averages: Vec> = Vec::new(); + let mut old_totals: Vec> = Vec::new(); totals.push(self.total_xact_count.clone()); + old_totals.push(self.old_total_xact_count.clone()); averages.push(self.avg_xact_count.clone()); totals.push(self.total_query_count.clone()); + old_totals.push(self.old_total_query_count.clone()); averages.push(self.avg_query_count.clone()); totals.push(self.total_received.clone()); + old_totals.push(self.old_total_received.clone()); averages.push(self.avg_recv.clone()); totals.push(self.total_sent.clone()); + old_totals.push(self.old_total_sent.clone()); averages.push(self.avg_sent.clone()); totals.push(self.total_xact_time.clone()); + old_totals.push(self.old_total_xact_time.clone()); averages.push(self.avg_xact_time.clone()); totals.push(self.total_query_time.clone()); + old_totals.push(self.old_total_query_time.clone()); averages.push(self.avg_query_time.clone()); totals.push(self.total_wait_time.clone()); + old_totals.push(self.old_total_wait_time.clone()); averages.push(self.avg_wait_time.clone()); totals.push(self.total_errors.clone()); + old_totals.push(self.old_total_errors.clone()); averages.push(self.avg_errors.clone()); - (totals, averages) + (totals, averages, old_totals) } } From 36d56b95a30a3c7929a78ac91b4833ee792105b0 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 10 May 2023 10:24:35 -0500 Subject: [PATCH 402/551] Fix docker-compose dev stack for Apple silicon (#432) The docker-compose dev setup is broken under Apple silicon, starting the stack fails with the following error. Switching to a different docker image fixes the issue. --- dev/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 71704bcbc..5fda0c130 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -25,7 +25,7 @@ x-common-env-pg: services: main: - image: kubernetes/pause + image: gcr.io/google_containers/pause:3.2 ports: - 6432 @@ -64,7 +64,7 @@ services: <<: *common-env-pg POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5 PGPORT: 10432 - command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] + command: ["postgres", "-p", "10432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"] toxiproxy: build: . From 4440d2fb56389a99df7eb258c59bcc839a8e5f6a Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 11 May 2023 20:37:58 -0400 Subject: [PATCH 403/551] Fixes average stats bug (#436) * Add test * Fix test * Add fix --- src/stats.rs | 15 +++++++++++++-- src/stats/address.rs | 14 ++++++++------ src/stats/server.rs | 11 +++++++++++ tests/ruby/admin_spec.rb | 13 +++++++------ 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/stats.rs b/src/stats.rs index b95a143a2..6de784f5e 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -107,8 +107,19 @@ impl Collector { loop { interval.tick().await; - for stats in SERVER_STATS.read().values() { - stats.address_stats().update_averages(); + // Hold read lock for duration of update to retain all server stats + let server_stats = SERVER_STATS.read(); + + for stats in server_stats.values() { + if !stats.check_address_stat_average_is_updated_status() { + stats.address_stats().update_averages(); + stats.set_address_stat_average_is_updated_status(true); + } + } + + // Reset to false for next update + for stats in server_stats.values() { + stats.set_address_stat_average_is_updated_status(false); } } }); diff --git a/src/stats/address.rs b/src/stats/address.rs index 51d6a688e..89e4ebe78 100644 --- a/src/stats/address.rs +++ b/src/stats/address.rs @@ -30,6 +30,9 @@ pub struct AddressStats { pub avg_xact_time: Arc, pub avg_xact_count: Arc, pub avg_wait_time: Arc, + + // Determines if the averages have been updated since the last time they were reported + pub averages_updated: Arc, } impl IntoIterator for AddressStats { @@ -114,15 +117,14 @@ impl AddressStats { pub fn update_averages(&self) { let (totals, averages, old_totals) = self.fields_iterators(); - for data in itertools::izip!(totals, averages, old_totals) { - let (total, average, old_total) = data; - let total = total.load(Ordering::Relaxed); - let old = old_total.load(Ordering::Relaxed); + for (total, average, old_total) in itertools::izip!(totals, averages, old_totals) { + let total_value = total.load(Ordering::Relaxed); + let old_total_value = old_total.load(Ordering::Relaxed); average.store( - (total - old) / (crate::stats::STAT_PERIOD / 1_000), + (total_value - old_total_value) / (crate::stats::STAT_PERIOD / 1_000), Ordering::Relaxed, ); // Avg / second - old_total.store(total, Ordering::Relaxed); + old_total.store(total_value, Ordering::Relaxed); } } diff --git a/src/stats/server.rs b/src/stats/server.rs index d25f3b4eb..399e585f6 100644 --- a/src/stats/server.rs +++ b/src/stats/server.rs @@ -139,6 +139,17 @@ impl ServerStats { self.address.stats.clone() } + pub fn check_address_stat_average_is_updated_status(&self) -> bool { + self.address.stats.averages_updated.load(Ordering::Relaxed) + } + + pub fn set_address_stat_average_is_updated_status(&self, is_checked: bool) { + self.address + .stats + .averages_updated + .store(is_checked, Ordering::Relaxed); + } + // Helper methods for show_servers pub fn pool_name(&self) -> String { self.pool_stats.database() diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index ea21630f9..e054b45e2 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -14,11 +14,12 @@ describe "SHOW STATS" do context "clients connect and make one query" do it "updates *_query_time and *_wait_time" do - connection = PG::connect("#{pgcat_conn_str}?application_name=one_query") - connection.async_exec("SELECT pg_sleep(0.25)") - connection.async_exec("SELECT pg_sleep(0.25)") - connection.async_exec("SELECT pg_sleep(0.25)") - connection.close + connections = Array.new(3) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + Thread.new { c.async_exec("SELECT pg_sleep(0.25)") } + end + sleep(1) + connections.map(&:close) # wait for averages to be calculated, we shouldn't do this too often sleep(15.5) @@ -26,7 +27,7 @@ results = admin_conn.async_exec("SHOW STATS")[0] admin_conn.close expect(results["total_query_time"].to_i).to be_within(200).of(750) - expect(results["avg_query_time"].to_i).to_not eq(0) + expect(results["avg_query_time"].to_i).to be_within(20).of(50) expect(results["total_wait_time"].to_i).to_not eq(0) expect(results["avg_wait_time"].to_i).to_not eq(0) From fceb556b8b4a64f0697bed1ff28b33994b67db61 Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 11 May 2023 20:40:10 -0400 Subject: [PATCH 404/551] Improve logging for connection cleanup (#428) * initial commit * fix * fmt --- src/server.rs | 64 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/src/server.rs b/src/server.rs index 8f0ece999..dceab49d9 100644 --- a/src/server.rs +++ b/src/server.rs @@ -103,6 +103,48 @@ impl StreamInner { } } +#[derive(Copy, Clone)] +struct CleanupState { + /// If server connection requires DISCARD ALL before checkin because of set statement + needs_cleanup_set: bool, + + /// If server connection requires DISCARD ALL before checkin because of prepare statement + needs_cleanup_prepare: bool, +} + +impl CleanupState { + fn new() -> Self { + CleanupState { + needs_cleanup_set: false, + needs_cleanup_prepare: false, + } + } + + fn needs_cleanup(&self) -> bool { + self.needs_cleanup_set || self.needs_cleanup_prepare + } + + fn set_true(&mut self) { + self.needs_cleanup_set = true; + self.needs_cleanup_prepare = true; + } + + fn reset(&mut self) { + self.needs_cleanup_set = false; + self.needs_cleanup_prepare = false; + } +} + +impl std::fmt::Display for CleanupState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "SET: {}, PREPARE: {}", + self.needs_cleanup_set, self.needs_cleanup_prepare + ) + } +} + /// Server state. pub struct Server { /// Server host, e.g. localhost, @@ -131,8 +173,8 @@ pub struct Server { /// Is the server broken? We'll remote it from the pool if so. bad: bool, - /// If server connection requires a DISCARD ALL before checkin - needs_cleanup: bool, + /// If server connection requires DISCARD ALL before checkin + cleanup_state: CleanupState, /// Mapping of clients and servers used for query cancellation. client_server_map: ClientServerMap, @@ -630,7 +672,7 @@ impl Server { in_transaction: false, data_available: false, bad: false, - needs_cleanup: false, + cleanup_state: CleanupState::new(), client_server_map, addr_set, connected_at: chrono::offset::Utc::now().naive_utc(), @@ -793,12 +835,12 @@ impl Server { // This will reduce amount of discard statements sent if !self.in_transaction { debug!("Server connection marked for clean up"); - self.needs_cleanup = true; + self.cleanup_state.needs_cleanup_set = true; } } "PREPARE\0" => { debug!("Server connection marked for clean up"); - self.needs_cleanup = true; + self.cleanup_state.needs_cleanup_prepare = true; } _ => (), } @@ -960,11 +1002,11 @@ impl Server { // to avoid leaking state between clients. For performance reasons we only // send `DISCARD ALL` if we think the session is altered instead of just sending // it before each checkin. - if self.needs_cleanup { - warn!("Server returned with session state altered, discarding state"); + if self.cleanup_state.needs_cleanup() { + warn!("Server returned with session state altered, discarding state ({}) for application {}", self.cleanup_state, self.application_name); self.query("DISCARD ALL").await?; self.query("RESET ROLE").await?; - self.needs_cleanup = false; + self.cleanup_state.reset(); } Ok(()) @@ -976,12 +1018,12 @@ impl Server { self.application_name = name.to_string(); // We don't want `SET application_name` to mark the server connection // as needing cleanup - let needs_cleanup_before = self.needs_cleanup; + let needs_cleanup_before = self.cleanup_state; let result = Ok(self .query(&format!("SET application_name = '{}'", name)) .await?); - self.needs_cleanup = needs_cleanup_before; + self.cleanup_state = needs_cleanup_before; result } else { Ok(()) @@ -1006,7 +1048,7 @@ impl Server { // Marks a connection as needing DISCARD ALL at checkin pub fn mark_dirty(&mut self) { - self.needs_cleanup = true; + self.cleanup_state.set_true(); } pub fn mirror_send(&mut self, bytes: &BytesMut) { From e1d4e504d4bf122d2903bcb4e0bb18650dbc50e3 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 12 May 2023 09:50:52 -0700 Subject: [PATCH 405/551] Prewarmer (#435) * Prewarmer * hmm * Tests * default * fix test * Correct configuration * Added minimal config example * remove connect_timeout --- Cargo.lock | 2 +- Cargo.toml | 2 +- pgcat.minimal.toml | 22 ++++ pgcat.toml | 72 ++++++++++- src/config.rs | 78 +++++++++++- src/mirrors.rs | 1 + src/plugins/intercept.rs | 194 ++--------------------------- src/plugins/mod.rs | 1 + src/plugins/prewarmer.rs | 28 +++++ src/plugins/query_logger.rs | 31 ++--- src/plugins/table_access.rs | 34 ++--- src/pool.rs | 66 +++++----- src/query_router.rs | 69 +++++++--- src/server.rs | 2 + tests/ruby/helpers/pgcat_helper.rb | 19 ++- 15 files changed, 337 insertions(+), 284 deletions(-) create mode 100644 pgcat.minimal.toml create mode 100644 src/plugins/prewarmer.rs diff --git a/Cargo.lock b/Cargo.lock index 9c553d458..4703a0b26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -897,7 +897,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pgcat" -version = "1.0.2-alpha1" +version = "1.0.2-alpha2" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 5428ab312..af1b9ae08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.0.2-alpha1" +version = "1.0.2-alpha2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/pgcat.minimal.toml b/pgcat.minimal.toml new file mode 100644 index 000000000..4b17a4549 --- /dev/null +++ b/pgcat.minimal.toml @@ -0,0 +1,22 @@ +# This is an example of the most basic config +# that will mimic what PgBouncer does in transaction mode with one server. + +[general] + +host = "0.0.0.0" +port = 6433 +admin_username = "pgcat" +admin_password = "pgcat" + +[pools.pgml.users.0] +username = "postgres" +password = "postgres" +pool_size = 10 +min_pool_size = 1 +pool_mode = "transaction" + +[pools.pgml.shards.0] +servers = [ + ["127.0.0.1", 28815, "primary"] +] +database = "postgres" diff --git a/pgcat.toml b/pgcat.toml index ce3663290..e6b54b25d 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -77,6 +77,58 @@ admin_username = "admin_user" # Password to access the virtual administrative database admin_password = "admin_pass" +# Default plugins that are configured on all pools. +[plugins] + +# Prewarmer plugin that runs queries on server startup, before giving the connection +# to the client. +[plugins.prewarmer] +enabled = false +queries = [ + "SELECT pg_prewarm('pgbench_accounts')", +] + +# Log all queries to stdout. +[plugins.query_logger] +enabled = false + +# Block access to tables that Postgres does not allow us to control. +[plugins.table_access] +enabled = false +tables = [ + "pg_user", + "pg_roles", + "pg_database", +] + +# Intercept user queries and give a fake reply. +[plugins.intercept] +enabled = true + +[plugins.intercept.queries.0] + +query = "select current_database() as a, current_schemas(false) as b" +schema = [ + ["a", "text"], + ["b", "text"], +] +result = [ + ["${DATABASE}", "{public}"], +] + +[plugins.intercept.queries.1] + +query = "select current_database(), current_schema(), current_user" +schema = [ + ["current_database", "text"], + ["current_schema", "text"], + ["current_user", "text"], +] +result = [ + ["${DATABASE}", "public", "${USER}"], +] + + # pool configs are structured as pool. # the pool_name is what clients use as database name when connecting. # For a pool named `sharded_db`, clients access that pool using connection string like @@ -154,12 +206,20 @@ connect_timeout = 3000 # Specifies how often (in seconds) cached ip addresses for servers are rechecked (see `dns_cache_enabled`). # dns_max_ttl = 30 -[plugins] +# Plugins can be configured on a pool-per-pool basis. This overrides the global plugins setting, +# so all plugins have to be configured here again. +[pool.sharded_db.plugins] -[plugins.query_logger] +[pools.sharded_db.plugins.prewarmer] +enabled = true +queries = [ + "SELECT pg_prewarm('pgbench_accounts')", +] + +[pools.sharded_db.plugins.query_logger] enabled = false -[plugins.table_access] +[pools.sharded_db.plugins.table_access] enabled = false tables = [ "pg_user", @@ -167,10 +227,10 @@ tables = [ "pg_database", ] -[plugins.intercept] +[pools.sharded_db.plugins.intercept] enabled = true -[plugins.intercept.queries.0] +[pools.sharded_db.plugins.intercept.queries.0] query = "select current_database() as a, current_schemas(false) as b" schema = [ @@ -181,7 +241,7 @@ result = [ ["${DATABASE}", "{public}"], ] -[plugins.intercept.queries.1] +[pools.sharded_db.plugins.intercept.queries.1] query = "select current_database(), current_schema(), current_user" schema = [ diff --git a/src/config.rs b/src/config.rs index c7fce635e..f41777305 100644 --- a/src/config.rs +++ b/src/config.rs @@ -122,6 +122,16 @@ impl Default for Address { } } +impl std::fmt::Display for Address { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "[address: {}:{}][database: {}][user: {}]", + self.host, self.port, self.database, self.username + ) + } +} + // We need to implement PartialEq by ourselves so we skip stats in the comparison impl PartialEq for Address { fn eq(&self, other: &Self) -> bool { @@ -235,6 +245,8 @@ pub struct General { pub port: u16, pub enable_prometheus_exporter: Option, + + #[serde(default = "General::default_prometheus_exporter_port")] pub prometheus_exporter_port: i16, #[serde(default = "General::default_connect_timeout")] @@ -374,6 +386,10 @@ impl General { pub fn default_validate_config() -> bool { true } + + pub fn default_prometheus_exporter_port() -> i16 { + 9930 + } } impl Default for General { @@ -462,6 +478,7 @@ pub struct Pool { #[serde(default = "Pool::default_load_balancing_mode")] pub load_balancing_mode: LoadBalancingMode, + #[serde(default = "Pool::default_default_role")] pub default_role: String, #[serde(default)] // False @@ -476,6 +493,7 @@ pub struct Pool { pub server_lifetime: Option, + #[serde(default = "Pool::default_sharding_function")] pub sharding_function: ShardingFunction, #[serde(default = "Pool::default_automatic_sharding_key")] @@ -489,6 +507,7 @@ pub struct Pool { pub auth_query_user: Option, pub auth_query_password: Option, + pub plugins: Option, pub shards: BTreeMap, pub users: BTreeMap, // Note, don't put simple fields below these configs. There's a compatibility issue with TOML that makes it @@ -521,6 +540,14 @@ impl Pool { None } + pub fn default_default_role() -> String { + "any".into() + } + + pub fn default_sharding_function() -> ShardingFunction { + ShardingFunction::PgBigintHash + } + pub fn validate(&mut self) -> Result<(), Error> { match self.default_role.as_ref() { "any" => (), @@ -609,6 +636,7 @@ impl Default for Pool { auth_query_user: None, auth_query_password: None, server_lifetime: None, + plugins: None, } } } @@ -687,30 +715,50 @@ impl Default for Shard { } } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, Hash, Eq)] pub struct Plugins { pub intercept: Option, pub table_access: Option, pub query_logger: Option, + pub prewarmer: Option, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +impl std::fmt::Display for Plugins { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "interceptor: {}, table_access: {}, query_logger: {}, prewarmer: {}", + self.intercept.is_some(), + self.table_access.is_some(), + self.query_logger.is_some(), + self.prewarmer.is_some(), + ) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, Hash, Eq)] pub struct Intercept { pub enabled: bool, pub queries: BTreeMap, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, Hash, Eq)] pub struct TableAccess { pub enabled: bool, pub tables: Vec, } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, Hash, Eq)] pub struct QueryLogger { pub enabled: bool, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, Hash, Eq)] +pub struct Prewarmer { + pub enabled: bool, + pub queries: Vec, +} + impl Intercept { pub fn substitute(&mut self, db: &str, user: &str) { for (_, query) in self.queries.iter_mut() { @@ -720,7 +768,7 @@ impl Intercept { } } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, Hash, Eq)] pub struct Query { pub query: String, pub schema: Vec>, @@ -754,8 +802,13 @@ pub struct Config { #[serde(default = "Config::default_path")] pub path: String, + // General and global settings. pub general: General, + + // Plugins that should run in all pools. pub plugins: Option, + + // Connection pools. pub pools: HashMap, } @@ -940,6 +993,13 @@ impl Config { "Server TLS certificate verification: {}", self.general.verify_server_certificate ); + info!( + "Plugins: {}", + match self.plugins { + Some(ref plugins) => plugins.to_string(), + None => "not configured".into(), + } + ); for (pool_name, pool_config) in &self.pools { // TODO: Make this output prettier (maybe a table?) @@ -1006,6 +1066,14 @@ impl Config { None => "default".to_string(), } ); + info!( + "[pool: {}] Plugins: {}", + pool_name, + match pool_config.plugins { + Some(ref plugins) => plugins.to_string(), + None => "not configured".into(), + } + ); for user in &pool_config.users { info!( diff --git a/src/mirrors.rs b/src/mirrors.rs index 17f91d4d1..d6d691ff4 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -43,6 +43,7 @@ impl MirroredClient { ClientServerMap::default(), Arc::new(PoolStats::new(identifier, cfg.clone())), Arc::new(RwLock::new(None)), + None, ); Pool::builder() diff --git a/src/plugins/intercept.rs b/src/plugins/intercept.rs index 88d24d0e4..166294bc2 100644 --- a/src/plugins/intercept.rs +++ b/src/plugins/intercept.rs @@ -2,52 +2,21 @@ //! //! It intercepts queries and returns fake results. -use arc_swap::ArcSwap; use async_trait::async_trait; use bytes::{BufMut, BytesMut}; -use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; use sqlparser::ast::Statement; -use std::collections::HashMap; -use log::{debug, info}; -use std::sync::Arc; +use log::debug; use crate::{ config::Intercept as InterceptConfig, errors::Error, messages::{command_complete, data_row_nullable, row_description, DataType}, plugins::{Plugin, PluginOutput}, - pool::{PoolIdentifier, PoolMap}, query_router::QueryRouter, }; -pub static CONFIG: Lazy>> = - Lazy::new(|| ArcSwap::from_pointee(HashMap::new())); - -/// Check if the interceptor plugin has been enabled. -pub fn enabled() -> bool { - !CONFIG.load().is_empty() -} - -pub fn setup(intercept_config: &InterceptConfig, pools: &PoolMap) { - let mut config = HashMap::new(); - for (identifier, _) in pools.iter() { - let mut intercept_config = intercept_config.clone(); - intercept_config.substitute(&identifier.db, &identifier.user); - config.insert(identifier.clone(), intercept_config); - } - - CONFIG.store(Arc::new(config)); - - info!("Intercepting {} queries", intercept_config.queries.len()); -} - -pub fn disable() { - CONFIG.store(Arc::new(HashMap::new())); -} - // TODO: use these structs for deserialization #[derive(Serialize, Deserialize)] pub struct Rule { @@ -63,33 +32,35 @@ pub struct Column { } /// The intercept plugin. -pub struct Intercept; +pub struct Intercept<'a> { + pub enabled: bool, + pub config: &'a InterceptConfig, +} #[async_trait] -impl Plugin for Intercept { +impl<'a> Plugin for Intercept<'a> { async fn run( &mut self, query_router: &QueryRouter, ast: &Vec, ) -> Result { - if ast.is_empty() { + if !self.enabled || ast.is_empty() { return Ok(PluginOutput::Allow); } - let mut result = BytesMut::new(); - let query_map = match CONFIG.load().get(&PoolIdentifier::new( + let mut config = self.config.clone(); + config.substitute( &query_router.pool_settings().db, &query_router.pool_settings().user.username, - )) { - Some(query_map) => query_map.clone(), - None => return Ok(PluginOutput::Allow), - }; + ); + + let mut result = BytesMut::new(); for q in ast { // Normalization let q = q.to_string().to_ascii_lowercase(); - for (_, target) in query_map.queries.iter() { + for (_, target) in config.queries.iter() { if target.query.as_str() == q { debug!("Intercepting query: {}", q); @@ -147,142 +118,3 @@ impl Plugin for Intercept { } } } - -/// Make IntelliJ SQL plugin believe it's talking to an actual database -/// instead of PgCat. -#[allow(dead_code)] -fn fool_datagrip(database: &str, user: &str) -> Value { - json!([ - { - "query": "select current_database() as a, current_schemas(false) as b", - "schema": [ - { - "name": "a", - "data_type": "text", - }, - { - "name": "b", - "data_type": "anyarray", - }, - ], - - "result": [ - [database, "{public}"], - ], - }, - { - "query": "select current_database(), current_schema(), current_user", - "schema": [ - { - "name": "current_database", - "data_type": "text", - }, - { - "name": "current_schema", - "data_type": "text", - }, - { - "name": "current_user", - "data_type": "text", - } - ], - - "result": [ - ["sharded_db", "public", "sharding_user"], - ], - }, - { - "query": "select cast(n.oid as bigint) as id, datname as name, d.description, datistemplate as is_template, datallowconn as allow_connections, pg_catalog.pg_get_userbyid(n.datdba) as \"owner\" from pg_catalog.pg_database as n left join pg_catalog.pg_shdescription as d on n.oid = d.objoid order by case when datname = pg_catalog.current_database() then -cast(1 as bigint) else cast(n.oid as bigint) end", - "schema": [ - { - "name": "id", - "data_type": "oid", - }, - { - "name": "name", - "data_type": "text", - }, - { - "name": "description", - "data_type": "text", - }, - { - "name": "is_template", - "data_type": "bool", - }, - { - "name": "allow_connections", - "data_type": "bool", - }, - { - "name": "owner", - "data_type": "text", - } - ], - "result": [ - ["16387", database, "", "f", "t", user], - ] - }, - { - "query": "select cast(r.oid as bigint) as role_id, rolname as role_name, rolsuper as is_super, rolinherit as is_inherit, rolcreaterole as can_createrole, rolcreatedb as can_createdb, rolcanlogin as can_login, rolreplication as is_replication, rolconnlimit as conn_limit, rolvaliduntil as valid_until, rolbypassrls as bypass_rls, rolconfig as config, d.description from pg_catalog.pg_roles as r left join pg_catalog.pg_shdescription as d on d.objoid = r.oid", - "schema": [ - { - "name": "role_id", - "data_type": "oid", - }, - { - "name": "role_name", - "data_type": "text", - }, - { - "name": "is_super", - "data_type": "bool", - }, - { - "name": "is_inherit", - "data_type": "bool", - }, - { - "name": "can_createrole", - "data_type": "bool", - }, - { - "name": "can_createdb", - "data_type": "bool", - }, - { - "name": "can_login", - "data_type": "bool", - }, - { - "name": "is_replication", - "data_type": "bool", - }, - { - "name": "conn_limit", - "data_type": "int4", - }, - { - "name": "valid_until", - "data_type": "text", - }, - { - "name": "bypass_rls", - "data_type": "bool", - }, - { - "name": "config", - "data_type": "text", - }, - { - "name": "description", - "data_type": "text", - }, - ], - "result": [ - ["10", "postgres", "f", "t", "f", "f", "t", "f", "-1", "", "f", "", ""], - ["16419", user, "f", "t", "f", "f", "t", "f", "-1", "", "f", "", ""], - ] - } - ]) -} diff --git a/src/plugins/mod.rs b/src/plugins/mod.rs index 6661ece63..5ef6009a5 100644 --- a/src/plugins/mod.rs +++ b/src/plugins/mod.rs @@ -9,6 +9,7 @@ //! pub mod intercept; +pub mod prewarmer; pub mod query_logger; pub mod table_access; diff --git a/src/plugins/prewarmer.rs b/src/plugins/prewarmer.rs new file mode 100644 index 000000000..a09bbe9dd --- /dev/null +++ b/src/plugins/prewarmer.rs @@ -0,0 +1,28 @@ +//! Prewarm new connections before giving them to the client. +use crate::{errors::Error, server::Server}; +use log::info; + +pub struct Prewarmer<'a> { + pub enabled: bool, + pub server: &'a mut Server, + pub queries: &'a Vec, +} + +impl<'a> Prewarmer<'a> { + pub async fn run(&mut self) -> Result<(), Error> { + if !self.enabled { + return Ok(()); + } + + for query in self.queries { + info!( + "{} Prewarning with query: `{}`", + self.server.address(), + query + ); + self.server.query(&query).await?; + } + + Ok(()) + } +} diff --git a/src/plugins/query_logger.rs b/src/plugins/query_logger.rs index 2dfda8bcf..debdf39fa 100644 --- a/src/plugins/query_logger.rs +++ b/src/plugins/query_logger.rs @@ -5,44 +5,33 @@ use crate::{ plugins::{Plugin, PluginOutput}, query_router::QueryRouter, }; -use arc_swap::ArcSwap; use async_trait::async_trait; use log::info; -use once_cell::sync::Lazy; use sqlparser::ast::Statement; -use std::sync::Arc; -static ENABLED: Lazy> = Lazy::new(|| ArcSwap::from_pointee(false)); - -pub struct QueryLogger; - -pub fn setup() { - ENABLED.store(Arc::new(true)); - - info!("Logging queries to stdout"); -} - -pub fn disable() { - ENABLED.store(Arc::new(false)); -} - -pub fn enabled() -> bool { - **ENABLED.load() +pub struct QueryLogger<'a> { + pub enabled: bool, + pub user: &'a str, + pub db: &'a str, } #[async_trait] -impl Plugin for QueryLogger { +impl<'a> Plugin for QueryLogger<'a> { async fn run( &mut self, _query_router: &QueryRouter, ast: &Vec, ) -> Result { + if !self.enabled { + return Ok(PluginOutput::Allow); + } + let query = ast .iter() .map(|q| q.to_string()) .collect::>() .join("; "); - info!("{}", query); + info!("[pool: {}][user: {}] {}", self.user, self.db, query); Ok(PluginOutput::Allow) } diff --git a/src/plugins/table_access.rs b/src/plugins/table_access.rs index 4613a4fb3..79c1260e4 100644 --- a/src/plugins/table_access.rs +++ b/src/plugins/table_access.rs @@ -5,53 +5,39 @@ use async_trait::async_trait; use sqlparser::ast::{visit_relations, Statement}; use crate::{ - config::TableAccess as TableAccessConfig, errors::Error, plugins::{Plugin, PluginOutput}, query_router::QueryRouter, }; -use log::{debug, info}; +use log::debug; -use arc_swap::ArcSwap; use core::ops::ControlFlow; -use once_cell::sync::Lazy; -use std::sync::Arc; -static CONFIG: Lazy>> = Lazy::new(|| ArcSwap::from_pointee(vec![])); - -pub fn setup(config: &TableAccessConfig) { - CONFIG.store(Arc::new(config.tables.clone())); - - info!("Blocking access to {} tables", config.tables.len()); +pub struct TableAccess<'a> { + pub enabled: bool, + pub tables: &'a Vec, } -pub fn enabled() -> bool { - !CONFIG.load().is_empty() -} - -pub fn disable() { - CONFIG.store(Arc::new(vec![])); -} - -pub struct TableAccess; - #[async_trait] -impl Plugin for TableAccess { +impl<'a> Plugin for TableAccess<'a> { async fn run( &mut self, _query_router: &QueryRouter, ast: &Vec, ) -> Result { + if !self.enabled { + return Ok(PluginOutput::Allow); + } + let mut found = None; - let forbidden_tables = CONFIG.load(); visit_relations(ast, |relation| { let relation = relation.to_string(); let parts = relation.split(".").collect::>(); let table_name = parts.last().unwrap(); - if forbidden_tables.contains(&table_name.to_string()) { + if self.tables.contains(&table_name.to_string()) { found = Some(table_name.to_string()); ControlFlow::<()>::Break(()) } else { diff --git a/src/pool.rs b/src/pool.rs index 4664193c7..8e03ae449 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -17,10 +17,13 @@ use std::sync::{ use std::time::Instant; use tokio::sync::Notify; -use crate::config::{get_config, Address, General, LoadBalancingMode, PoolMode, Role, User}; +use crate::config::{ + get_config, Address, General, LoadBalancingMode, Plugins, PoolMode, Role, User, +}; use crate::errors::Error; use crate::auth_passthrough::AuthPassthrough; +use crate::plugins::prewarmer; use crate::server::Server; use crate::sharding::ShardingFunction; use crate::stats::{AddressStats, ClientStats, PoolStats, ServerStats}; @@ -132,6 +135,9 @@ pub struct PoolSettings { pub auth_query: Option, pub auth_query_user: Option, pub auth_query_password: Option, + + /// Plugins + pub plugins: Option, } impl Default for PoolSettings { @@ -156,6 +162,7 @@ impl Default for PoolSettings { auth_query: None, auth_query_user: None, auth_query_password: None, + plugins: None, } } } @@ -195,6 +202,7 @@ pub struct ConnectionPool { paused: Arc, paused_waiter: Arc, + /// Statistics. pub stats: Arc, /// AuthInfo @@ -352,6 +360,10 @@ impl ConnectionPool { client_server_map.clone(), pool_stats.clone(), pool_auth_hash.clone(), + match pool_config.plugins { + Some(ref plugins) => Some(plugins.clone()), + None => config.plugins.clone(), + }, ); let connect_timeout = match pool_config.connect_timeout { @@ -377,7 +389,10 @@ impl ConnectionPool { .min() .unwrap(); - debug!("Pool reaper rate: {}ms", reaper_rate); + debug!( + "[pool: {}][user: {}] Pool reaper rate: {}ms", + pool_name, user.username, reaper_rate + ); let pool = Pool::builder() .max_size(user.pool_size) @@ -450,6 +465,10 @@ impl ConnectionPool { auth_query: pool_config.auth_query.clone(), auth_query_user: pool_config.auth_query_user.clone(), auth_query_password: pool_config.auth_query_password.clone(), + plugins: match pool_config.plugins { + Some(ref plugins) => Some(plugins.clone()), + None => config.plugins.clone(), + }, }, validated: Arc::new(AtomicBool::new(false)), paused: Arc::new(AtomicBool::new(false)), @@ -471,32 +490,6 @@ impl ConnectionPool { } } - if let Some(ref plugins) = config.plugins { - if let Some(ref intercept) = plugins.intercept { - if intercept.enabled { - crate::plugins::intercept::setup(intercept, &new_pools); - } else { - crate::plugins::intercept::disable(); - } - } - - if let Some(ref table_access) = plugins.table_access { - if table_access.enabled { - crate::plugins::table_access::setup(table_access); - } else { - crate::plugins::table_access::disable(); - } - } - - if let Some(ref query_logger) = plugins.query_logger { - if query_logger.enabled { - crate::plugins::query_logger::setup(); - } else { - crate::plugins::query_logger::disable(); - } - } - } - POOLS.store(Arc::new(new_pools.clone())); Ok(()) } @@ -923,6 +916,7 @@ pub struct ServerPool { client_server_map: ClientServerMap, stats: Arc, auth_hash: Arc>>, + plugins: Option, } impl ServerPool { @@ -933,6 +927,7 @@ impl ServerPool { client_server_map: ClientServerMap, stats: Arc, auth_hash: Arc>>, + plugins: Option, ) -> ServerPool { ServerPool { address, @@ -941,6 +936,7 @@ impl ServerPool { client_server_map, stats, auth_hash, + plugins, } } } @@ -973,7 +969,19 @@ impl ManageConnection for ServerPool { ) .await { - Ok(conn) => { + Ok(mut conn) => { + if let Some(ref plugins) = self.plugins { + if let Some(ref prewarmer) = plugins.prewarmer { + let mut prewarmer = prewarmer::Prewarmer { + enabled: prewarmer.enabled, + server: &mut conn, + queries: &prewarmer.queries, + }; + + prewarmer.run().await?; + } + } + stats.idle(); Ok(conn) } diff --git a/src/query_router.rs b/src/query_router.rs index d995b804f..3e3a23a8f 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -15,10 +15,7 @@ use sqlparser::parser::Parser; use crate::config::Role; use crate::errors::Error; use crate::messages::BytesMutReader; -use crate::plugins::{ - intercept, query_logger, table_access, Intercept, Plugin, PluginOutput, QueryLogger, - TableAccess, -}; +use crate::plugins::{Intercept, Plugin, PluginOutput, QueryLogger, TableAccess}; use crate::pool::PoolSettings; use crate::sharding::Sharder; @@ -793,13 +790,27 @@ impl QueryRouter { /// Add your plugins here and execute them. pub async fn execute_plugins(&self, ast: &Vec) -> Result { - if query_logger::enabled() { - let mut query_logger = QueryLogger {}; + let plugins = match self.pool_settings.plugins { + Some(ref plugins) => plugins, + None => return Ok(PluginOutput::Allow), + }; + + if let Some(ref query_logger) = plugins.query_logger { + let mut query_logger = QueryLogger { + enabled: query_logger.enabled, + user: &self.pool_settings.user.username, + db: &self.pool_settings.db, + }; + let _ = query_logger.run(&self, ast).await; } - if intercept::enabled() { - let mut intercept = Intercept {}; + if let Some(ref intercept) = plugins.intercept { + let mut intercept = Intercept { + enabled: intercept.enabled, + config: &intercept, + }; + let result = intercept.run(&self, ast).await; if let Ok(PluginOutput::Intercept(output)) = result { @@ -807,8 +818,12 @@ impl QueryRouter { } } - if table_access::enabled() { - let mut table_access = TableAccess {}; + if let Some(ref table_access) = plugins.table_access { + let mut table_access = TableAccess { + enabled: table_access.enabled, + tables: &table_access.tables, + }; + let result = table_access.run(&self, ast).await; if let Ok(PluginOutput::Deny(error)) = result { @@ -1161,6 +1176,7 @@ mod test { auth_query_password: None, auth_query_user: None, db: "test".to_string(), + plugins: None, }; let mut qr = QueryRouter::new(); assert_eq!(qr.active_role, None); @@ -1235,7 +1251,9 @@ mod test { auth_query_password: None, auth_query_user: None, db: "test".to_string(), + plugins: None, }; + let mut qr = QueryRouter::new(); qr.update_pool_settings(pool_settings.clone()); @@ -1379,17 +1397,25 @@ mod test { #[tokio::test] async fn test_table_access_plugin() { - use crate::config::TableAccess; - let ta = TableAccess { + use crate::config::{Plugins, TableAccess}; + let table_access = TableAccess { enabled: true, tables: vec![String::from("pg_database")], }; - - crate::plugins::table_access::setup(&ta); + let plugins = Plugins { + table_access: Some(table_access), + intercept: None, + query_logger: None, + prewarmer: None, + }; QueryRouter::setup(); + let mut pool_settings = PoolSettings::default(); + pool_settings.query_parser_enabled = true; + pool_settings.plugins = Some(plugins); - let qr = QueryRouter::new(); + let mut qr = QueryRouter::new(); + qr.update_pool_settings(pool_settings); let query = simple_query("SELECT * FROM pg_database"); let ast = QueryRouter::parse(&query).unwrap(); @@ -1403,4 +1429,17 @@ mod test { )) ); } + + #[tokio::test] + async fn test_plugins_disabled_by_defaault() { + QueryRouter::setup(); + let qr = QueryRouter::new(); + + let query = simple_query("SELECT * FROM pg_database"); + let ast = QueryRouter::parse(&query).unwrap(); + + let res = qr.execute_plugins(&ast).await; + + assert_eq!(res, Ok(PluginOutput::Allow)); + } } diff --git a/src/server.rs b/src/server.rs index dceab49d9..244c06e75 100644 --- a/src/server.rs +++ b/src/server.rs @@ -970,6 +970,8 @@ impl Server { /// It will use the simple query protocol. /// Result will not be returned, so this is useful for things like `SET` or `ROLLBACK`. pub async fn query(&mut self, query: &str) -> Result<(), Error> { + debug!("Running `{}` on server {:?}", query, self.address); + let query = simple_query(query); self.send(&query).await?; diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index ad4c32a47..e36801eb1 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -41,7 +41,24 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mod "1" => { "database" => "shard1", "servers" => [["localhost", primary1.port.to_s, "primary"]] }, "2" => { "database" => "shard2", "servers" => [["localhost", primary2.port.to_s, "primary"]] }, }, - "users" => { "0" => user } + "users" => { "0" => user }, + "plugins" => { + "intercept" => { + "enabled" => true, + "queries" => { + "0" => { + "query" => "select current_database() as a, current_schemas(false) as b", + "schema" => [ + ["a", "text"], + ["b", "text"], + ], + "result" => [ + ["${DATABASE}", "{public}"], + ] + } + } + } + } } } pgcat.update_config(pgcat_cfg) From 78e349f49be4fc9cef6b07be83a0a6b5d9531d46 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 12 May 2023 12:48:37 -0700 Subject: [PATCH 406/551] Allow to deploy pools without checking (#438) --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/pool.rs | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4703a0b26..d89fa79f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -897,7 +897,7 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pgcat" -version = "1.0.2-alpha2" +version = "1.0.2-alpha3" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index af1b9ae08..17f7e0e73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.0.2-alpha2" +version = "1.0.2-alpha3" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/pool.rs b/src/pool.rs index 8e03ae449..6235e2209 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -401,9 +401,13 @@ impl ConnectionPool { .idle_timeout(Some(std::time::Duration::from_millis(idle_timeout))) .max_lifetime(Some(std::time::Duration::from_millis(server_lifetime))) .reaper_rate(std::time::Duration::from_millis(reaper_rate)) - .test_on_check_out(false) - .build(manager) - .await?; + .test_on_check_out(false); + + let pool = if config.general.validate_config { + pool.build(manager).await? + } else { + pool.build_unchecked(manager) + }; pools.push(pool); servers.push(address); From 973655ffdfd18e1cae52d128e726bf057f2ba2a4 Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 18 May 2023 00:38:10 -0400 Subject: [PATCH 407/551] Fix time based average stats (#442) * keep track of current stats and zero them after updating averages * Try tests * typo * remove commented test stuff * Avoid dividing by zero * Fix test * refactor, get rid of iterator. do it manually * trigger build * Fix --- src/stats.rs | 1 + src/stats/address.rs | 233 ++++++++++++++++++++++++--------------- src/stats/server.rs | 31 ++---- tests/ruby/admin_spec.rb | 2 +- 4 files changed, 151 insertions(+), 116 deletions(-) diff --git a/src/stats.rs b/src/stats.rs index 6de784f5e..ce076d2dd 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -113,6 +113,7 @@ impl Collector { for stats in server_stats.values() { if !stats.check_address_stat_average_is_updated_status() { stats.address_stats().update_averages(); + stats.address_stats().reset_current_counts(); stats.set_address_stat_average_is_updated_status(true); } } diff --git a/src/stats/address.rs b/src/stats/address.rs index 89e4ebe78..a04864452 100644 --- a/src/stats/address.rs +++ b/src/stats/address.rs @@ -1,35 +1,26 @@ use std::sync::atomic::*; use std::sync::Arc; +#[derive(Debug, Clone, Default)] +struct AddressStatFields { + xact_count: Arc, + query_count: Arc, + bytes_received: Arc, + bytes_sent: Arc, + xact_time: Arc, + query_time: Arc, + wait_time: Arc, + errors: Arc, +} + /// Internal address stats #[derive(Debug, Clone, Default)] pub struct AddressStats { - pub total_xact_count: Arc, - pub total_query_count: Arc, - pub total_received: Arc, - pub total_sent: Arc, - pub total_xact_time: Arc, - pub total_query_time: Arc, - pub total_wait_time: Arc, - pub total_errors: Arc, - - pub old_total_xact_count: Arc, - pub old_total_query_count: Arc, - pub old_total_received: Arc, - pub old_total_sent: Arc, - pub old_total_xact_time: Arc, - pub old_total_query_time: Arc, - pub old_total_wait_time: Arc, - pub old_total_errors: Arc, - - pub avg_query_count: Arc, - pub avg_query_time: Arc, - pub avg_recv: Arc, - pub avg_sent: Arc, - pub avg_errors: Arc, - pub avg_xact_time: Arc, - pub avg_xact_count: Arc, - pub avg_wait_time: Arc, + total: AddressStatFields, + + current: AddressStatFields, + + averages: AddressStatFields, // Determines if the averages have been updated since the last time they were reported pub averages_updated: Arc, @@ -43,67 +34,67 @@ impl IntoIterator for AddressStats { vec![ ( "total_xact_count".to_string(), - self.total_xact_count.load(Ordering::Relaxed), + self.total.xact_count.load(Ordering::Relaxed), ), ( "total_query_count".to_string(), - self.total_query_count.load(Ordering::Relaxed), + self.total.query_count.load(Ordering::Relaxed), ), ( "total_received".to_string(), - self.total_received.load(Ordering::Relaxed), + self.total.bytes_received.load(Ordering::Relaxed), ), ( "total_sent".to_string(), - self.total_sent.load(Ordering::Relaxed), + self.total.bytes_sent.load(Ordering::Relaxed), ), ( "total_xact_time".to_string(), - self.total_xact_time.load(Ordering::Relaxed), + self.total.xact_time.load(Ordering::Relaxed), ), ( "total_query_time".to_string(), - self.total_query_time.load(Ordering::Relaxed), + self.total.query_time.load(Ordering::Relaxed), ), ( "total_wait_time".to_string(), - self.total_wait_time.load(Ordering::Relaxed), + self.total.wait_time.load(Ordering::Relaxed), ), ( "total_errors".to_string(), - self.total_errors.load(Ordering::Relaxed), + self.total.errors.load(Ordering::Relaxed), ), ( "avg_xact_count".to_string(), - self.avg_xact_count.load(Ordering::Relaxed), + self.averages.xact_count.load(Ordering::Relaxed), ), ( "avg_query_count".to_string(), - self.avg_query_count.load(Ordering::Relaxed), + self.averages.query_count.load(Ordering::Relaxed), ), ( "avg_recv".to_string(), - self.avg_recv.load(Ordering::Relaxed), + self.averages.bytes_received.load(Ordering::Relaxed), ), ( "avg_sent".to_string(), - self.avg_sent.load(Ordering::Relaxed), + self.averages.bytes_sent.load(Ordering::Relaxed), ), ( "avg_errors".to_string(), - self.avg_errors.load(Ordering::Relaxed), + self.averages.errors.load(Ordering::Relaxed), ), ( "avg_xact_time".to_string(), - self.avg_xact_time.load(Ordering::Relaxed), + self.averages.xact_time.load(Ordering::Relaxed), ), ( "avg_query_time".to_string(), - self.avg_query_time.load(Ordering::Relaxed), + self.averages.query_time.load(Ordering::Relaxed), ), ( "avg_wait_time".to_string(), - self.avg_wait_time.load(Ordering::Relaxed), + self.averages.wait_time.load(Ordering::Relaxed), ), ] .into_iter() @@ -111,21 +102,120 @@ impl IntoIterator for AddressStats { } impl AddressStats { + pub fn xact_count_add(&self) { + self.total.xact_count.fetch_add(1, Ordering::Relaxed); + self.current.xact_count.fetch_add(1, Ordering::Relaxed); + } + + pub fn query_count_add(&self) { + self.total.query_count.fetch_add(1, Ordering::Relaxed); + self.current.query_count.fetch_add(1, Ordering::Relaxed); + } + + pub fn bytes_received_add(&self, bytes: u64) { + self.total + .bytes_received + .fetch_add(bytes, Ordering::Relaxed); + self.current + .bytes_received + .fetch_add(bytes, Ordering::Relaxed); + } + + pub fn bytes_sent_add(&self, bytes: u64) { + self.total.bytes_sent.fetch_add(bytes, Ordering::Relaxed); + self.current.bytes_sent.fetch_add(bytes, Ordering::Relaxed); + } + + pub fn xact_time_add(&self, time: u64) { + self.total.xact_time.fetch_add(time, Ordering::Relaxed); + self.current.xact_time.fetch_add(time, Ordering::Relaxed); + } + + pub fn query_time_add(&self, time: u64) { + self.total.query_time.fetch_add(time, Ordering::Relaxed); + self.current.query_time.fetch_add(time, Ordering::Relaxed); + } + + pub fn wait_time_add(&self, time: u64) { + self.total.wait_time.fetch_add(time, Ordering::Relaxed); + self.current.wait_time.fetch_add(time, Ordering::Relaxed); + } + pub fn error(&self) { - self.total_errors.fetch_add(1, Ordering::Relaxed); + self.total.errors.fetch_add(1, Ordering::Relaxed); + self.current.errors.fetch_add(1, Ordering::Relaxed); } pub fn update_averages(&self) { - let (totals, averages, old_totals) = self.fields_iterators(); - for (total, average, old_total) in itertools::izip!(totals, averages, old_totals) { - let total_value = total.load(Ordering::Relaxed); - let old_total_value = old_total.load(Ordering::Relaxed); - average.store( - (total_value - old_total_value) / (crate::stats::STAT_PERIOD / 1_000), - Ordering::Relaxed, - ); // Avg / second - old_total.store(total_value, Ordering::Relaxed); + let stat_period_per_second = crate::stats::STAT_PERIOD / 1_000; + + // xact_count + let current_xact_count = self.current.xact_count.load(Ordering::Relaxed); + let current_xact_time = self.current.xact_time.load(Ordering::Relaxed); + self.averages.xact_count.store( + current_xact_count / stat_period_per_second, + Ordering::Relaxed, + ); + if current_xact_count == 0 { + self.averages.xact_time.store(0, Ordering::Relaxed); + } else { + self.averages + .xact_time + .store(current_xact_time / current_xact_count, Ordering::Relaxed); + } + + // query_count + let current_query_count = self.current.query_count.load(Ordering::Relaxed); + let current_query_time = self.current.query_time.load(Ordering::Relaxed); + self.averages.query_count.store( + current_query_count / stat_period_per_second, + Ordering::Relaxed, + ); + if current_query_count == 0 { + self.averages.query_time.store(0, Ordering::Relaxed); + } else { + self.averages + .query_time + .store(current_query_time / current_query_count, Ordering::Relaxed); } + + // bytes_received + let current_bytes_received = self.current.bytes_received.load(Ordering::Relaxed); + self.averages.bytes_received.store( + current_bytes_received / stat_period_per_second, + Ordering::Relaxed, + ); + + // bytes_sent + let current_bytes_sent = self.current.bytes_sent.load(Ordering::Relaxed); + self.averages.bytes_sent.store( + current_bytes_sent / stat_period_per_second, + Ordering::Relaxed, + ); + + // wait_time + let current_wait_time = self.current.wait_time.load(Ordering::Relaxed); + self.averages.wait_time.store( + current_wait_time / stat_period_per_second, + Ordering::Relaxed, + ); + + // errors + let current_errors = self.current.errors.load(Ordering::Relaxed); + self.averages + .errors + .store(current_errors / stat_period_per_second, Ordering::Relaxed); + } + + pub fn reset_current_counts(&self) { + self.current.xact_count.store(0, Ordering::Relaxed); + self.current.xact_time.store(0, Ordering::Relaxed); + self.current.query_count.store(0, Ordering::Relaxed); + self.current.query_time.store(0, Ordering::Relaxed); + self.current.bytes_received.store(0, Ordering::Relaxed); + self.current.bytes_sent.store(0, Ordering::Relaxed); + self.current.wait_time.store(0, Ordering::Relaxed); + self.current.errors.store(0, Ordering::Relaxed); } pub fn populate_row(&self, row: &mut Vec) { @@ -133,43 +223,4 @@ impl AddressStats { row.push(value.to_string()); } } - - fn fields_iterators( - &self, - ) -> ( - Vec>, - Vec>, - Vec>, - ) { - let mut totals: Vec> = Vec::new(); - let mut averages: Vec> = Vec::new(); - let mut old_totals: Vec> = Vec::new(); - - totals.push(self.total_xact_count.clone()); - old_totals.push(self.old_total_xact_count.clone()); - averages.push(self.avg_xact_count.clone()); - totals.push(self.total_query_count.clone()); - old_totals.push(self.old_total_query_count.clone()); - averages.push(self.avg_query_count.clone()); - totals.push(self.total_received.clone()); - old_totals.push(self.old_total_received.clone()); - averages.push(self.avg_recv.clone()); - totals.push(self.total_sent.clone()); - old_totals.push(self.old_total_sent.clone()); - averages.push(self.avg_sent.clone()); - totals.push(self.total_xact_time.clone()); - old_totals.push(self.old_total_xact_time.clone()); - averages.push(self.avg_xact_time.clone()); - totals.push(self.total_query_time.clone()); - old_totals.push(self.old_total_query_time.clone()); - averages.push(self.avg_query_time.clone()); - totals.push(self.total_wait_time.clone()); - old_totals.push(self.old_total_wait_time.clone()); - averages.push(self.avg_wait_time.clone()); - totals.push(self.total_errors.clone()); - old_totals.push(self.old_total_errors.clone()); - averages.push(self.avg_errors.clone()); - - (totals, averages, old_totals) - } } diff --git a/src/stats/server.rs b/src/stats/server.rs index 399e585f6..a327fa340 100644 --- a/src/stats/server.rs +++ b/src/stats/server.rs @@ -177,12 +177,9 @@ impl ServerStats { } pub fn checkout_time(&self, microseconds: u64, application_name: String) { - // Update server stats and address aggergation stats + // Update server stats and address aggregation stats self.set_application(application_name); - self.address - .stats - .total_wait_time - .fetch_add(microseconds, Ordering::Relaxed); + self.address.stats.wait_time_add(microseconds); self.pool_stats .maxwait .fetch_max(microseconds, Ordering::Relaxed); @@ -191,13 +188,8 @@ impl ServerStats { /// Report a query executed by a client against a server pub fn query(&self, milliseconds: u64, application_name: &str) { self.set_application(application_name.to_string()); - let address_stats = self.address_stats(); - address_stats - .total_query_count - .fetch_add(1, Ordering::Relaxed); - address_stats - .total_query_time - .fetch_add(milliseconds, Ordering::Relaxed); + self.address.stats.query_count_add(); + self.address.stats.query_time_add(milliseconds); } /// Report a transaction executed by a client a server @@ -208,29 +200,20 @@ impl ServerStats { self.set_application(application_name.to_string()); self.transaction_count.fetch_add(1, Ordering::Relaxed); - self.address - .stats - .total_xact_count - .fetch_add(1, Ordering::Relaxed); + self.address.stats.xact_count_add(); } /// Report data sent to a server pub fn data_sent(&self, amount_bytes: usize) { self.bytes_sent .fetch_add(amount_bytes as u64, Ordering::Relaxed); - self.address - .stats - .total_sent - .fetch_add(amount_bytes as u64, Ordering::Relaxed); + self.address.stats.bytes_sent_add(amount_bytes as u64); } /// Report data received from a server pub fn data_received(&self, amount_bytes: usize) { self.bytes_received .fetch_add(amount_bytes as u64, Ordering::Relaxed); - self.address - .stats - .total_received - .fetch_add(amount_bytes as u64, Ordering::Relaxed); + self.address.stats.bytes_received_add(amount_bytes as u64); } } diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index e054b45e2..f93b1a6c8 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -27,7 +27,7 @@ results = admin_conn.async_exec("SHOW STATS")[0] admin_conn.close expect(results["total_query_time"].to_i).to be_within(200).of(750) - expect(results["avg_query_time"].to_i).to be_within(20).of(50) + expect(results["avg_query_time"].to_i).to be_within(50).of(250) expect(results["total_wait_time"].to_i).to_not eq(0) expect(results["avg_wait_time"].to_i).to_not eq(0) From 603cbedc369ac1c20c3e58f49f9bbc2ea00fda42 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 18 May 2023 10:46:55 -0700 Subject: [PATCH 408/551] Optionally clean up server connections (#444) * Optionally clean up server connections * move setting to pool * fix test * Print setting to screen * fmt * Fix pool_settings override in tests --- src/config.rs | 17 +++++++++++ src/mirrors.rs | 1 + src/pool.rs | 20 +++++++++++++ src/server.rs | 12 ++++++-- tests/ruby/helpers/pgcat_helper.rb | 46 ++++++++++++++++-------------- tests/ruby/misc_spec.rb | 25 ++++++++++++++++ 6 files changed, 97 insertions(+), 24 deletions(-) diff --git a/src/config.rs b/src/config.rs index f41777305..2f5596c60 100644 --- a/src/config.rs +++ b/src/config.rs @@ -487,10 +487,15 @@ pub struct Pool { #[serde(default)] // False pub primary_reads_enabled: bool, + /// Maximum time to allow for establishing a new server connection. pub connect_timeout: Option, + /// Close idle connections that have been opened for longer than this. pub idle_timeout: Option, + /// Close server connections that have been opened for longer than this. + /// Only applied to idle connections. If the connection is actively used for + /// longer than this period, the pool will not interrupt it. pub server_lifetime: Option, #[serde(default = "Pool::default_sharding_function")] @@ -507,6 +512,9 @@ pub struct Pool { pub auth_query_user: Option, pub auth_query_password: Option, + #[serde(default = "Pool::default_cleanup_server_connections")] + pub cleanup_server_connections: bool, + pub plugins: Option, pub shards: BTreeMap, pub users: BTreeMap, @@ -548,6 +556,10 @@ impl Pool { ShardingFunction::PgBigintHash } + pub fn default_cleanup_server_connections() -> bool { + true + } + pub fn validate(&mut self) -> Result<(), Error> { match self.default_role.as_ref() { "any" => (), @@ -637,6 +649,7 @@ impl Default for Pool { auth_query_password: None, server_lifetime: None, plugins: None, + cleanup_server_connections: true, } } } @@ -1066,6 +1079,10 @@ impl Config { None => "default".to_string(), } ); + info!( + "[pool: {}] Cleanup server connections: {}", + pool_name, pool_config.cleanup_server_connections + ); info!( "[pool: {}] Plugins: {}", pool_name, diff --git a/src/mirrors.rs b/src/mirrors.rs index d6d691ff4..7e2c9a097 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -44,6 +44,7 @@ impl MirroredClient { Arc::new(PoolStats::new(identifier, cfg.clone())), Arc::new(RwLock::new(None)), None, + true, ); Pool::builder() diff --git a/src/pool.rs b/src/pool.rs index 6235e2209..a0b0c4d13 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -364,6 +364,7 @@ impl ConnectionPool { Some(ref plugins) => Some(plugins.clone()), None => config.plugins.clone(), }, + pool_config.cleanup_server_connections, ); let connect_timeout = match pool_config.connect_timeout { @@ -914,13 +915,29 @@ impl ConnectionPool { /// Wrapper for the bb8 connection pool. pub struct ServerPool { + /// Server address. address: Address, + + /// Server Postgres user. user: User, + + /// Server database. database: String, + + /// Client/server mapping. client_server_map: ClientServerMap, + + /// Server statistics. stats: Arc, + + /// Server auth hash (for auth passthrough). auth_hash: Arc>>, + + /// Server plugins. plugins: Option, + + /// Should we clean up dirty connections before putting them into the pool? + cleanup_connections: bool, } impl ServerPool { @@ -932,6 +949,7 @@ impl ServerPool { stats: Arc, auth_hash: Arc>>, plugins: Option, + cleanup_connections: bool, ) -> ServerPool { ServerPool { address, @@ -941,6 +959,7 @@ impl ServerPool { stats, auth_hash, plugins, + cleanup_connections, } } } @@ -970,6 +989,7 @@ impl ManageConnection for ServerPool { self.client_server_map.clone(), stats.clone(), self.auth_hash.clone(), + self.cleanup_connections, ) .await { diff --git a/src/server.rs b/src/server.rs index 244c06e75..32dd91f8b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -188,13 +188,16 @@ pub struct Server { /// Application name using the server at the moment. application_name: String, - // Last time that a successful server send or response happened + /// Last time that a successful server send or response happened last_activity: SystemTime, mirror_manager: Option, - // Associated addresses used + /// Associated addresses used addr_set: Option, + + /// Should clean up dirty connections? + cleanup_connections: bool, } impl Server { @@ -207,6 +210,7 @@ impl Server { client_server_map: ClientServerMap, stats: Arc, auth_hash: Arc>>, + cleanup_connections: bool, ) -> Result { let cached_resolver = CACHED_RESOLVER.load(); let mut addr_set: Option = None; @@ -687,6 +691,7 @@ impl Server { address.mirrors.clone(), )), }, + cleanup_connections, }; server.set_name("pgcat").await?; @@ -1004,7 +1009,7 @@ impl Server { // to avoid leaking state between clients. For performance reasons we only // send `DISCARD ALL` if we think the session is altered instead of just sending // it before each checkin. - if self.cleanup_state.needs_cleanup() { + if self.cleanup_state.needs_cleanup() && self.cleanup_connections { warn!("Server returned with session state altered, discarding state ({}) for application {}", self.cleanup_state, self.application_name); self.query("DISCARD ALL").await?; self.query("RESET ROLE").await?; @@ -1084,6 +1089,7 @@ impl Server { client_server_map, Arc::new(ServerStats::default()), Arc::new(RwLock::new(None)), + true, ) .await?; debug!("Connected!, sending query."); diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index e36801eb1..7a5bd71d6 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -118,7 +118,7 @@ def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction", lb end end - def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random", log_level="info") + def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mode="random", log_level="info", pool_settings={}) user = { "password" => "sharding_user", "pool_size" => pool_size, @@ -134,28 +134,32 @@ def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mo replica1 = PgInstance.new(8432, user["username"], user["password"], "shard0") replica2 = PgInstance.new(9432, user["username"], user["password"], "shard0") + pool_config = { + "default_role" => "any", + "pool_mode" => pool_mode, + "load_balancing_mode" => lb_mode, + "primary_reads_enabled" => false, + "query_parser_enabled" => false, + "sharding_function" => "pg_bigint_hash", + "shards" => { + "0" => { + "database" => "shard0", + "servers" => [ + ["localhost", primary.port.to_s, "primary"], + ["localhost", replica0.port.to_s, "replica"], + ["localhost", replica1.port.to_s, "replica"], + ["localhost", replica2.port.to_s, "replica"] + ] + }, + }, + "users" => { "0" => user } + } + + pool_config = pool_config.merge(pool_settings) + # Main proxy configs pgcat_cfg["pools"] = { - "#{pool_name}" => { - "default_role" => "any", - "pool_mode" => pool_mode, - "load_balancing_mode" => lb_mode, - "primary_reads_enabled" => false, - "query_parser_enabled" => false, - "sharding_function" => "pg_bigint_hash", - "shards" => { - "0" => { - "database" => "shard0", - "servers" => [ - ["localhost", primary.port.to_s, "primary"], - ["localhost", replica0.port.to_s, "replica"], - ["localhost", replica1.port.to_s, "replica"], - ["localhost", replica2.port.to_s, "replica"] - ] - }, - }, - "users" => { "0" => user } - } + "#{pool_name}" => pool_config, } pgcat_cfg["general"]["port"] = pgcat.port pgcat.update_config(pgcat_cfg) diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index e4d6f6fbb..fe216e5bf 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -320,6 +320,31 @@ expect(processes.primary.count_query("DISCARD ALL")).to eq(0) end end + + context "server cleanup disabled" do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 1, "transaction", "random", "info", { "cleanup_server_connections" => false }) } + + it "will not clean up connection state" do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + processes.primary.reset_stats + conn.async_exec("SET statement_timeout TO 1000") + conn.close + + puts processes.pgcat.logs + expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + end + + it "will not clean up prepared statements" do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + processes.primary.reset_stats + conn.async_exec("PREPARE prepared_q (int) AS SELECT $1") + + conn.close + + puts processes.pgcat.logs + expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + end + end end describe "Idle client timeout" do From 5abd23fecf6d6a1d64b7050491782bc33341fec4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 18 May 2023 16:41:22 -0700 Subject: [PATCH 409/551] Ensure data makes it to the client (#446) * Ensure data makes it to the client * flush all buffers --- src/client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client.rs b/src/client.rs index 331a0da40..24d76bd57 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1313,7 +1313,7 @@ where .receive_server_message(server, &address, &pool, &self.stats.clone()) .await?; - match write_all_half(&mut self.write, &response).await { + match write_all_flush(&mut self.write, &response).await { Ok(_) => (), Err(err) => { server.mark_bad(); @@ -1408,7 +1408,7 @@ where .receive_server_message(server, address, pool, client_stats) .await?; - match write_all_half(&mut self.write, &response).await { + match write_all_flush(&mut self.write, &response).await { Ok(_) => (), Err(err) => { server.mark_bad(); From bcecb77141a4807499f82a8c4c1b1e91421e1764 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 May 2023 08:58:13 -0700 Subject: [PATCH 410/551] chore(deps): bump toml from 0.7.3 to 0.7.4 (#447) Bumps [toml](https://github.com/toml-rs/toml) from 0.7.3 to 0.7.4. - [Commits](https://github.com/toml-rs/toml/compare/toml-v0.7.3...toml-v0.7.4) --- updated-dependencies: - dependency-name: toml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d89fa79f4..d26c4d52b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1252,9 +1252,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" dependencies = [ "serde", ] @@ -1528,9 +1528,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" dependencies = [ "serde", "serde_spanned", @@ -1540,18 +1540,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.6" +version = "0.19.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08de71aa0d6e348f070457f85af8bd566e2bc452156a423ddf22861b3a953fae" +checksum = "92d964908cec0d030b812013af25a0e57fddfadb1e066ecc6681d86253129d4f" dependencies = [ "indexmap", "serde", @@ -1905,9 +1905,9 @@ checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winnow" -version = "0.3.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" dependencies = [ "memchr", ] From cc92e919430ed4db92fafbba17f077c894bd7991 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 23 May 2023 08:44:49 -0500 Subject: [PATCH 411/551] Refactor Pool Stats to be based off of Server/Client stats (#445) What is wrong Stats reported by SHOW POOLS seem to be leaking. We see lingering cl_idle , cl_waiting, and similarly for sv_idle , sv_active. We confirmed that these are reporting issues not actual lingering clients. This behavior is readily reproducible by running while true; do psql "postgres://sharding_user:sharding_user@localhost:6432/sharded_db" -c "SELECT 1" > /dev/null 2>&1 & done Why it happens I wasn't able to get to figure our the reason for the bug but my best guess is that we have race conditions when updating pool-level stats. So even though individual update operations are atomic, we perform a check then update sequence which is not protected by a guard. https://github.com/postgresml/pgcat/blob/main/src/stats/pool.rs#L174-L179 I am also suspecting that using Relaxed ordering might allow this behavior (I changed all operations to use Ordering::SeqCst but still got lingering clients) How to fix Since SHOW POOLS/SHOW SERVER/SHOW CLIENTS all show the current state of the proxy (as opposed to SHOW STATS which show aggregate values), this PR refactors SHOW POOLS to have it construct the results directly from SHOW SERVER and SHOW CLIENT datasets. This reduces the complexity of stat updates and eliminates the need for having locks when updating pool stats as we only care about updating individual client/server states. This will change the semantics of maxwait, so instead of it holding the maxwait time ever encountered by a client (connected or disconnected), it will only consider connected clients which should be okay given PgCat tends to hold on to client connections more than Pgbouncer. --- src/admin.rs | 40 +---- src/client.rs | 14 +- src/mirrors.rs | 8 +- src/pool.rs | 42 ++--- src/prometheus.rs | 24 ++- src/stats.rs | 21 --- src/stats/client.rs | 20 +-- src/stats/pool.rs | 377 +++++++++++++-------------------------- src/stats/server.rs | 24 +-- tests/ruby/admin_spec.rb | 322 +-------------------------------- tests/ruby/stats_spec.rb | 369 ++++++++++++++++++++++++++++++++++++++ 11 files changed, 548 insertions(+), 713 deletions(-) create mode 100644 tests/ruby/stats_spec.rb diff --git a/src/admin.rs b/src/admin.rs index ceba20c86..804476c90 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,4 +1,5 @@ use crate::pool::BanReason; +use crate::stats::pool::PoolStats; use bytes::{Buf, BufMut, BytesMut}; use log::{error, info, trace}; use nix::sys::signal::{self, Signal}; @@ -14,7 +15,7 @@ use crate::errors::Error; use crate::messages::*; use crate::pool::ClientServerMap; use crate::pool::{get_all_pools, get_pool}; -use crate::stats::{get_client_stats, get_pool_stats, get_server_stats, ClientState, ServerState}; +use crate::stats::{get_client_stats, get_server_stats, ClientState, ServerState}; pub fn generate_server_info_for_admin() -> BytesMut { let mut server_info = BytesMut::new(); @@ -254,39 +255,12 @@ async fn show_pools(stream: &mut T) -> Result<(), Error> where T: tokio::io::AsyncWrite + std::marker::Unpin, { - let all_pool_stats = get_pool_stats(); - - let columns = vec![ - ("database", DataType::Text), - ("user", DataType::Text), - ("pool_mode", DataType::Text), - ("cl_idle", DataType::Numeric), - ("cl_active", DataType::Numeric), - ("cl_waiting", DataType::Numeric), - ("cl_cancel_req", DataType::Numeric), - ("sv_active", DataType::Numeric), - ("sv_idle", DataType::Numeric), - ("sv_used", DataType::Numeric), - ("sv_tested", DataType::Numeric), - ("sv_login", DataType::Numeric), - ("maxwait", DataType::Numeric), - ("maxwait_us", DataType::Numeric), - ]; - + let pool_lookup = PoolStats::construct_pool_lookup(); let mut res = BytesMut::new(); - res.put(row_description(&columns)); - - for ((_user_pool, _pool), pool_stats) in all_pool_stats { - let mut row = vec![ - pool_stats.database(), - pool_stats.user(), - pool_stats.pool_mode().to_string(), - ]; - pool_stats.populate_row(&mut row); - pool_stats.clear_maxwait(); - res.put(data_row(&row)); - } - + res.put(row_description(&PoolStats::generate_header())); + pool_lookup.iter().for_each(|(_identifier, pool_stats)| { + res.put(data_row(&pool_stats.generate_row())); + }); res.put(command_complete("SHOW")); // ReadyForQuery diff --git a/src/client.rs b/src/client.rs index 24d76bd57..1ff558b5a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -20,7 +20,7 @@ use crate::plugins::PluginOutput; use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; use crate::query_router::{Command, QueryRouter}; use crate::server::Server; -use crate::stats::{ClientStats, PoolStats, ServerStats}; +use crate::stats::{ClientStats, ServerStats}; use crate::tls::Tls; use tokio_rustls::server::TlsStream; @@ -654,24 +654,12 @@ where ready_for_query(&mut write).await?; trace!("Startup OK"); - let pool_stats = match get_pool(pool_name, username) { - Some(pool) => { - if !admin { - pool.stats - } else { - Arc::new(PoolStats::default()) - } - } - None => Arc::new(PoolStats::default()), - }; - let stats = Arc::new(ClientStats::new( process_id, application_name, username, pool_name, tokio::time::Instant::now(), - pool_stats, )); Ok(Client { diff --git a/src/mirrors.rs b/src/mirrors.rs index 7e2c9a097..0f2b02c01 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -7,8 +7,7 @@ use bytes::{Bytes, BytesMut}; use parking_lot::RwLock; use crate::config::{get_config, Address, Role, User}; -use crate::pool::{ClientServerMap, PoolIdentifier, ServerPool}; -use crate::stats::PoolStats; +use crate::pool::{ClientServerMap, ServerPool}; use log::{error, info, trace, warn}; use tokio::sync::mpsc::{channel, Receiver, Sender}; @@ -24,7 +23,7 @@ impl MirroredClient { async fn create_pool(&self) -> Pool { let config = get_config(); let default = std::time::Duration::from_millis(10_000).as_millis() as u64; - let (connection_timeout, idle_timeout, cfg) = + let (connection_timeout, idle_timeout, _cfg) = match config.pools.get(&self.address.pool_name) { Some(cfg) => ( cfg.connect_timeout.unwrap_or(default), @@ -34,14 +33,11 @@ impl MirroredClient { None => (default, default, crate::config::Pool::default()), }; - let identifier = PoolIdentifier::new(&self.database, &self.user.username); - let manager = ServerPool::new( self.address.clone(), self.user.clone(), self.database.as_str(), ClientServerMap::default(), - Arc::new(PoolStats::new(identifier, cfg.clone())), Arc::new(RwLock::new(None)), None, true, diff --git a/src/pool.rs b/src/pool.rs index a0b0c4d13..4fa277534 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -10,6 +10,7 @@ use rand::seq::SliceRandom; use rand::thread_rng; use regex::Regex; use std::collections::HashMap; +use std::fmt::{Display, Formatter}; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -26,7 +27,7 @@ use crate::auth_passthrough::AuthPassthrough; use crate::plugins::prewarmer; use crate::server::Server; use crate::sharding::ShardingFunction; -use crate::stats::{AddressStats, ClientStats, PoolStats, ServerStats}; +use crate::stats::{AddressStats, ClientStats, ServerStats}; pub type ProcessId = i32; pub type SecretKey = i32; @@ -76,6 +77,12 @@ impl PoolIdentifier { } } +impl Display for PoolIdentifier { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}@{}", self.user, self.db) + } +} + impl From<&Address> for PoolIdentifier { fn from(address: &Address) -> PoolIdentifier { PoolIdentifier::new(&address.database, &address.username) @@ -202,9 +209,6 @@ pub struct ConnectionPool { paused: Arc, paused_waiter: Arc, - /// Statistics. - pub stats: Arc, - /// AuthInfo pub auth_hash: Arc>>, } @@ -254,10 +258,6 @@ impl ConnectionPool { .clone() .into_keys() .collect::>(); - let pool_stats = Arc::new(PoolStats::new(identifier, pool_config.clone())); - - // Allow the pool to be seen in statistics - pool_stats.register(pool_stats.clone()); // Sort by shard number to ensure consistency. shard_ids.sort_by_key(|k| k.parse::().unwrap()); @@ -358,7 +358,6 @@ impl ConnectionPool { user.clone(), &shard.database, client_server_map.clone(), - pool_stats.clone(), pool_auth_hash.clone(), match pool_config.plugins { Some(ref plugins) => Some(plugins.clone()), @@ -429,7 +428,6 @@ impl ConnectionPool { let pool = ConnectionPool { databases: shards, - stats: pool_stats, addresses, banlist: Arc::new(RwLock::new(banlist)), config_hash: new_pool_hash_value, @@ -610,6 +608,10 @@ impl ConnectionPool { }); } + // Indicate we're waiting on a server connection from a pool. + let now = Instant::now(); + client_stats.waiting(); + while !candidates.is_empty() { // Get the next candidate let address = match candidates.pop() { @@ -628,10 +630,6 @@ impl ConnectionPool { } } - // Indicate we're waiting on a server connection from a pool. - let now = Instant::now(); - client_stats.waiting(); - // Check if we can connect let mut conn = match self.databases[address.shard][address.address_index] .get() @@ -669,7 +667,7 @@ impl ConnectionPool { .stats() .checkout_time(checkout_time, client_stats.application_name()); server.stats().active(client_stats.application_name()); - + client_stats.active(); return Ok((conn, address.clone())); } @@ -677,11 +675,19 @@ impl ConnectionPool { .run_health_check(address, server, now, client_stats) .await { + let checkout_time: u64 = now.elapsed().as_micros() as u64; + client_stats.checkout_time(checkout_time); + server + .stats() + .checkout_time(checkout_time, client_stats.application_name()); + server.stats().active(client_stats.application_name()); + client_stats.active(); return Ok((conn, address.clone())); } else { continue; } } + client_stats.idle(); Err(Error::AllServersDown) } @@ -927,9 +933,6 @@ pub struct ServerPool { /// Client/server mapping. client_server_map: ClientServerMap, - /// Server statistics. - stats: Arc, - /// Server auth hash (for auth passthrough). auth_hash: Arc>>, @@ -946,7 +949,6 @@ impl ServerPool { user: User, database: &str, client_server_map: ClientServerMap, - stats: Arc, auth_hash: Arc>>, plugins: Option, cleanup_connections: bool, @@ -956,7 +958,6 @@ impl ServerPool { user: user.clone(), database: database.to_string(), client_server_map, - stats, auth_hash, plugins, cleanup_connections, @@ -975,7 +976,6 @@ impl ManageConnection for ServerPool { let stats = Arc::new(ServerStats::new( self.address.clone(), - self.stats.clone(), tokio::time::Instant::now(), )); diff --git a/src/prometheus.rs b/src/prometheus.rs index 6e578bf07..b4884604a 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -9,8 +9,9 @@ use std::sync::atomic::Ordering; use std::sync::Arc; use crate::config::Address; -use crate::pool::get_all_pools; -use crate::stats::{get_pool_stats, get_server_stats, ServerStats}; +use crate::pool::{get_all_pools, PoolIdentifier}; +use crate::stats::pool::PoolStats; +use crate::stats::{get_server_stats, ServerStats}; struct MetricHelpType { help: &'static str, @@ -233,10 +234,10 @@ impl PrometheusMetric { Self::from_name(&format!("stats_{}", name), value, labels) } - fn from_pool(pool: &(String, String), name: &str, value: u64) -> Option> { + fn from_pool(pool_id: PoolIdentifier, name: &str, value: u64) -> Option> { let mut labels = HashMap::new(); - labels.insert("pool", pool.0.clone()); - labels.insert("user", pool.1.clone()); + labels.insert("pool", pool_id.db); + labels.insert("user", pool_id.user); Self::from_name(&format!("pools_{}", name), value, labels) } @@ -284,18 +285,15 @@ fn push_address_stats(lines: &mut Vec) { // Adds relevant metrics shown in a SHOW POOLS admin command. fn push_pool_stats(lines: &mut Vec) { - let pool_stats = get_pool_stats(); - for (pool, stats) in pool_stats.iter() { - let stats = &**stats; + let pool_stats = PoolStats::construct_pool_lookup(); + for (pool_id, stats) in pool_stats.iter() { for (name, value) in stats.clone() { - if let Some(prometheus_metric) = PrometheusMetric::::from_pool(pool, &name, value) + if let Some(prometheus_metric) = + PrometheusMetric::::from_pool(pool_id.clone(), &name, value) { lines.push(prometheus_metric.to_string()); } else { - warn!( - "Metric {} not implemented for ({},{})", - name, pool.0, pool.1 - ); + warn!("Metric {} not implemented for ({})", name, *pool_id); } } } diff --git a/src/stats.rs b/src/stats.rs index ce076d2dd..29c00d649 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -1,4 +1,3 @@ -use crate::pool::PoolIdentifier; /// Statistics and reporting. use arc_swap::ArcSwap; @@ -16,13 +15,11 @@ pub mod pool; pub mod server; pub use address::AddressStats; pub use client::{ClientState, ClientStats}; -pub use pool::PoolStats; pub use server::{ServerState, ServerStats}; /// Convenience types for various stats type ClientStatesLookup = HashMap>; type ServerStatesLookup = HashMap>; -type PoolStatsLookup = HashMap<(String, String), Arc>; /// Stats for individual client connections /// Used in SHOW CLIENTS. @@ -34,11 +31,6 @@ static CLIENT_STATS: Lazy>> = static SERVER_STATS: Lazy>> = Lazy::new(|| Arc::new(RwLock::new(ServerStatesLookup::default()))); -/// Aggregate stats for each pool (a pool is identified by database name and username) -/// Used in SHOW POOLS. -static POOL_STATS: Lazy>> = - Lazy::new(|| Arc::new(RwLock::new(PoolStatsLookup::default()))); - /// The statistics reporter. An instance is given to each possible source of statistics, /// e.g. client stats, server stats, connection pool stats. pub static REPORTER: Lazy> = @@ -80,13 +72,6 @@ impl Reporter { fn server_disconnecting(&self, server_id: i32) { SERVER_STATS.write().remove(&server_id); } - - /// Register a pool with the stats system. - fn pool_register(&self, identifier: PoolIdentifier, stats: Arc) { - POOL_STATS - .write() - .insert((identifier.db, identifier.user), stats); - } } /// The statistics collector which used for calculating averages @@ -139,12 +124,6 @@ pub fn get_server_stats() -> ServerStatesLookup { SERVER_STATS.read().clone() } -/// Get a snapshot of pool statistics. -/// by the `Collector`. -pub fn get_pool_stats() -> PoolStatsLookup { - POOL_STATS.read().clone() -} - /// Get the statistics reporter used to update stats across the pools/clients. pub fn get_reporter() -> Reporter { (*(*REPORTER.load())).clone() diff --git a/src/stats/client.rs b/src/stats/client.rs index 4cbcab2f7..6a30ec15a 100644 --- a/src/stats/client.rs +++ b/src/stats/client.rs @@ -1,4 +1,3 @@ -use super::PoolStats; use super::{get_reporter, Reporter}; use atomic_enum::atomic_enum; use std::sync::atomic::*; @@ -34,12 +33,14 @@ pub struct ClientStats { pool_name: String, connect_time: Instant, - pool_stats: Arc, reporter: Reporter, /// Total time spent waiting for a connection from pool, measures in microseconds pub total_wait_time: Arc, + /// Maximum time spent waiting for a connection from pool, measures in microseconds + pub max_wait_time: Arc, + /// Current state of the client pub state: Arc, @@ -61,8 +62,8 @@ impl Default for ClientStats { application_name: String::new(), username: String::new(), pool_name: String::new(), - pool_stats: Arc::new(PoolStats::default()), total_wait_time: Arc::new(AtomicU64::new(0)), + max_wait_time: Arc::new(AtomicU64::new(0)), state: Arc::new(AtomicClientState::new(ClientState::Idle)), transaction_count: Arc::new(AtomicU64::new(0)), query_count: Arc::new(AtomicU64::new(0)), @@ -79,11 +80,9 @@ impl ClientStats { username: &str, pool_name: &str, connect_time: Instant, - pool_stats: Arc, ) -> Self { Self { client_id, - pool_stats, connect_time, application_name: application_name.to_string(), username: username.to_string(), @@ -96,8 +95,6 @@ impl ClientStats { /// update metrics on the corresponding pool. pub fn disconnect(&self) { self.reporter.client_disconnecting(self.client_id); - self.pool_stats - .client_disconnect(self.state.load(Ordering::Relaxed)) } /// Register a client with the stats system. The stats system uses client_id @@ -105,27 +102,20 @@ impl ClientStats { pub fn register(&self, stats: Arc) { self.reporter.client_register(self.client_id, stats); self.state.store(ClientState::Idle, Ordering::Relaxed); - self.pool_stats.cl_idle.fetch_add(1, Ordering::Relaxed); } /// Reports a client is done querying the server and is no longer assigned a server connection pub fn idle(&self) { - self.pool_stats - .client_idle(self.state.load(Ordering::Relaxed)); self.state.store(ClientState::Idle, Ordering::Relaxed); } /// Reports a client is waiting for a connection pub fn waiting(&self) { - self.pool_stats - .client_waiting(self.state.load(Ordering::Relaxed)); self.state.store(ClientState::Waiting, Ordering::Relaxed); } /// Reports a client is done waiting for a connection and is about to query the server. pub fn active(&self) { - self.pool_stats - .client_active(self.state.load(Ordering::Relaxed)); self.state.store(ClientState::Active, Ordering::Relaxed); } @@ -144,6 +134,8 @@ impl ClientStats { pub fn checkout_time(&self, microseconds: u64) { self.total_wait_time .fetch_add(microseconds, Ordering::Relaxed); + self.max_wait_time + .fetch_max(microseconds, Ordering::Relaxed); } /// Report a query executed by a client against a server diff --git a/src/stats/pool.rs b/src/stats/pool.rs index 1b01ef2eb..d3ac78e9f 100644 --- a/src/stats/pool.rs +++ b/src/stats/pool.rs @@ -1,274 +1,151 @@ -use crate::config::Pool; -use crate::config::PoolMode; -use crate::pool::PoolIdentifier; -use std::sync::atomic::*; -use std::sync::Arc; +use log::debug; -use super::get_reporter; -use super::Reporter; use super::{ClientState, ServerState}; +use crate::{config::PoolMode, messages::DataType, pool::PoolIdentifier}; +use std::collections::HashMap; +use std::sync::atomic::*; -#[derive(Debug, Clone, Default)] +use crate::pool::get_all_pools; + +#[derive(Debug, Clone)] /// A struct that holds information about a Pool . pub struct PoolStats { - // Pool identifier, cannot be changed after creating the instance - identifier: PoolIdentifier, - - // Pool Config, cannot be changed after creating the instance - config: Pool, - - // A reference to the global reporter. - reporter: Reporter, - - /// Counters (atomics) - pub cl_idle: Arc, - pub cl_active: Arc, - pub cl_waiting: Arc, - pub cl_cancel_req: Arc, - pub sv_active: Arc, - pub sv_idle: Arc, - pub sv_used: Arc, - pub sv_tested: Arc, - pub sv_login: Arc, - pub maxwait: Arc, + pub identifier: PoolIdentifier, + pub mode: PoolMode, + pub cl_idle: u64, + pub cl_active: u64, + pub cl_waiting: u64, + pub cl_cancel_req: u64, + pub sv_active: u64, + pub sv_idle: u64, + pub sv_used: u64, + pub sv_tested: u64, + pub sv_login: u64, + pub maxwait: u64, } - -impl IntoIterator for PoolStats { - type Item = (String, u64); - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - vec![ - ("cl_idle".to_string(), self.cl_idle.load(Ordering::Relaxed)), - ( - "cl_active".to_string(), - self.cl_active.load(Ordering::Relaxed), - ), - ( - "cl_waiting".to_string(), - self.cl_waiting.load(Ordering::Relaxed), - ), - ( - "cl_cancel_req".to_string(), - self.cl_cancel_req.load(Ordering::Relaxed), - ), - ( - "sv_active".to_string(), - self.sv_active.load(Ordering::Relaxed), - ), - ("sv_idle".to_string(), self.sv_idle.load(Ordering::Relaxed)), - ("sv_used".to_string(), self.sv_used.load(Ordering::Relaxed)), - ( - "sv_tested".to_string(), - self.sv_tested.load(Ordering::Relaxed), - ), - ( - "sv_login".to_string(), - self.sv_login.load(Ordering::Relaxed), - ), - ( - "maxwait".to_string(), - self.maxwait.load(Ordering::Relaxed) / 1_000_000, - ), - ( - "maxwait_us".to_string(), - self.maxwait.load(Ordering::Relaxed) % 1_000_000, - ), - ] - .into_iter() - } -} - impl PoolStats { - pub fn new(identifier: PoolIdentifier, config: Pool) -> Self { - Self { + pub fn new(identifier: PoolIdentifier, mode: PoolMode) -> Self { + PoolStats { identifier, - config, - reporter: get_reporter(), - ..Default::default() - } - } - - // Getters - pub fn register(&self, stats: Arc) { - self.reporter.pool_register(self.identifier.clone(), stats); - } - - pub fn database(&self) -> String { - self.identifier.db.clone() - } - - pub fn user(&self) -> String { - self.identifier.user.clone() - } - - pub fn pool_mode(&self) -> PoolMode { - self.config.pool_mode - } - - /// Populates an array of strings with counters (used by admin in show pools) - pub fn populate_row(&self, row: &mut Vec) { - for (_key, value) in self.clone() { - row.push(value.to_string()); - } - } - - /// Deletes the maxwait counter, this is done everytime we obtain metrics - pub fn clear_maxwait(&self) { - self.maxwait.store(0, Ordering::Relaxed); - } - - /// Notified when a server of the pool enters login state. - /// - /// Arguments: - /// - /// `from`: The state of the server that notifies. - pub fn server_login(&self, from: ServerState) { - self.sv_login.fetch_add(1, Ordering::Relaxed); - if from != ServerState::Login { - self.decrease_from_server_state(from); - } - } - - /// Notified when a server of the pool become 'active' - /// - /// Arguments: - /// - /// `from`: The state of the server that notifies. - pub fn server_active(&self, from: ServerState) { - self.sv_active.fetch_add(1, Ordering::Relaxed); - if from != ServerState::Active { - self.decrease_from_server_state(from); - } - } - - /// Notified when a server of the pool become 'tested' - /// - /// Arguments: - /// - /// `from`: The state of the server that notifies. - pub fn server_tested(&self, from: ServerState) { - self.sv_tested.fetch_add(1, Ordering::Relaxed); - if from != ServerState::Tested { - self.decrease_from_server_state(from); + mode, + cl_idle: 0, + cl_active: 0, + cl_waiting: 0, + cl_cancel_req: 0, + sv_active: 0, + sv_idle: 0, + sv_used: 0, + sv_tested: 0, + sv_login: 0, + maxwait: 0, } } - /// Notified when a server of the pool become 'idle' - /// - /// Arguments: - /// - /// `from`: The state of the server that notifies. - pub fn server_idle(&self, from: ServerState) { - self.sv_idle.fetch_add(1, Ordering::Relaxed); - if from != ServerState::Idle { - self.decrease_from_server_state(from); - } - } + pub fn construct_pool_lookup() -> HashMap { + let mut map: HashMap = HashMap::new(); + let client_map = super::get_client_stats(); + let server_map = super::get_server_stats(); - /// Notified when a client of the pool become 'waiting' - /// - /// Arguments: - /// - /// `from`: The state of the client that notifies. - pub fn client_waiting(&self, from: ClientState) { - if from != ClientState::Waiting { - self.cl_waiting.fetch_add(1, Ordering::Relaxed); - self.decrease_from_client_state(from); + for (identifier, pool) in get_all_pools() { + map.insert( + identifier.clone(), + PoolStats::new(identifier, pool.settings.pool_mode), + ); } - } - /// Notified when a client of the pool become 'active' - /// - /// Arguments: - /// - /// `from`: The state of the client that notifies. - pub fn client_active(&self, from: ClientState) { - if from != ClientState::Active { - self.cl_active.fetch_add(1, Ordering::Relaxed); - self.decrease_from_client_state(from); + for client in client_map.values() { + match map.get_mut(&PoolIdentifier { + db: client.pool_name(), + user: client.username(), + }) { + Some(pool_stats) => { + match client.state.load(Ordering::Relaxed) { + ClientState::Active => pool_stats.cl_active += 1, + ClientState::Idle => pool_stats.cl_idle += 1, + ClientState::Waiting => pool_stats.cl_waiting += 1, + } + let max_wait = client.max_wait_time.load(Ordering::Relaxed); + pool_stats.maxwait = std::cmp::max(pool_stats.maxwait, max_wait); + } + None => debug!("Client from an obselete pool"), + } } - } - /// Notified when a client of the pool become 'idle' - /// - /// Arguments: - /// - /// `from`: The state of the client that notifies. - pub fn client_idle(&self, from: ClientState) { - if from != ClientState::Idle { - self.cl_idle.fetch_add(1, Ordering::Relaxed); - self.decrease_from_client_state(from); + for server in server_map.values() { + match map.get_mut(&PoolIdentifier { + db: server.pool_name(), + user: server.username(), + }) { + Some(pool_stats) => match server.state.load(Ordering::Relaxed) { + ServerState::Active => pool_stats.sv_active += 1, + ServerState::Idle => pool_stats.sv_idle += 1, + ServerState::Login => pool_stats.sv_login += 1, + ServerState::Tested => pool_stats.sv_tested += 1, + }, + None => debug!("Server from an obselete pool"), + } } - } - /// Notified when a client disconnects. - /// - /// Arguments: - /// - /// `from`: The state of the client that notifies. - pub fn client_disconnect(&self, from: ClientState) { - let counter = match from { - ClientState::Idle => &self.cl_idle, - ClientState::Waiting => &self.cl_waiting, - ClientState::Active => &self.cl_active, - }; - - Self::decrease_counter(counter.clone()); - } - - /// Notified when a server disconnects. - /// - /// Arguments: - /// - /// `from`: The state of the client that notifies. - pub fn server_disconnect(&self, from: ServerState) { - let counter = match from { - ServerState::Active => &self.sv_active, - ServerState::Idle => &self.sv_idle, - ServerState::Login => &self.sv_login, - ServerState::Tested => &self.sv_tested, - }; - Self::decrease_counter(counter.clone()); - } - - // helpers for counter decrease - fn decrease_from_server_state(&self, from: ServerState) { - let counter = match from { - ServerState::Tested => &self.sv_tested, - ServerState::Active => &self.sv_active, - ServerState::Idle => &self.sv_idle, - ServerState::Login => &self.sv_login, - }; - Self::decrease_counter(counter.clone()); - } - - fn decrease_from_client_state(&self, from: ClientState) { - let counter = match from { - ClientState::Active => &self.cl_active, - ClientState::Idle => &self.cl_idle, - ClientState::Waiting => &self.cl_waiting, - }; - Self::decrease_counter(counter.clone()); - } - - fn decrease_counter(value: Arc) { - if value.load(Ordering::Relaxed) > 0 { - value.fetch_sub(1, Ordering::Relaxed); - } + return map; + } + + pub fn generate_header() -> Vec<(&'static str, DataType)> { + return vec![ + ("database", DataType::Text), + ("user", DataType::Text), + ("pool_mode", DataType::Text), + ("cl_idle", DataType::Numeric), + ("cl_active", DataType::Numeric), + ("cl_waiting", DataType::Numeric), + ("cl_cancel_req", DataType::Numeric), + ("sv_active", DataType::Numeric), + ("sv_idle", DataType::Numeric), + ("sv_used", DataType::Numeric), + ("sv_tested", DataType::Numeric), + ("sv_login", DataType::Numeric), + ("maxwait", DataType::Numeric), + ("maxwait_us", DataType::Numeric), + ]; + } + + pub fn generate_row(&self) -> Vec { + return vec![ + self.identifier.db.clone(), + self.identifier.user.clone(), + self.mode.to_string(), + self.cl_idle.to_string(), + self.cl_active.to_string(), + self.cl_waiting.to_string(), + self.cl_cancel_req.to_string(), + self.sv_active.to_string(), + self.sv_idle.to_string(), + self.sv_used.to_string(), + self.sv_tested.to_string(), + self.sv_login.to_string(), + (self.maxwait / 1_000_000).to_string(), + (self.maxwait % 1_000_000).to_string(), + ]; } } -#[cfg(test)] -mod test { - use super::*; +impl IntoIterator for PoolStats { + type Item = (String, u64); + type IntoIter = std::vec::IntoIter; - #[test] - fn test_decrease() { - let stat: PoolStats = PoolStats::default(); - stat.server_login(ServerState::Login); - stat.server_idle(ServerState::Login); - assert_eq!(stat.sv_login.load(Ordering::Relaxed), 0); - assert_eq!(stat.sv_idle.load(Ordering::Relaxed), 1); + fn into_iter(self) -> Self::IntoIter { + vec![ + ("cl_idle".to_string(), self.cl_idle), + ("cl_active".to_string(), self.cl_active), + ("cl_waiting".to_string(), self.cl_waiting), + ("cl_cancel_req".to_string(), self.cl_cancel_req), + ("sv_active".to_string(), self.sv_active), + ("sv_idle".to_string(), self.sv_idle), + ("sv_used".to_string(), self.sv_used), + ("sv_tested".to_string(), self.sv_tested), + ("sv_login".to_string(), self.sv_login), + ("maxwait".to_string(), self.maxwait / 1_000_000), + ("maxwait_us".to_string(), self.maxwait % 1_000_000), + ] + .into_iter() } } diff --git a/src/stats/server.rs b/src/stats/server.rs index a327fa340..e156ee094 100644 --- a/src/stats/server.rs +++ b/src/stats/server.rs @@ -1,5 +1,4 @@ use super::AddressStats; -use super::PoolStats; use super::{get_reporter, Reporter}; use crate::config::Address; use atomic_enum::atomic_enum; @@ -38,7 +37,6 @@ pub struct ServerStats { address: Address, connect_time: Instant, - pool_stats: Arc, reporter: Reporter, /// Data @@ -57,7 +55,6 @@ impl Default for ServerStats { server_id: 0, application_name: Arc::new(RwLock::new(String::new())), address: Address::default(), - pool_stats: Arc::new(PoolStats::default()), connect_time: Instant::now(), state: Arc::new(AtomicServerState::new(ServerState::Login)), bytes_sent: Arc::new(AtomicU64::new(0)), @@ -71,10 +68,9 @@ impl Default for ServerStats { } impl ServerStats { - pub fn new(address: Address, pool_stats: Arc, connect_time: Instant) -> Self { + pub fn new(address: Address, connect_time: Instant) -> Self { Self { address, - pool_stats, connect_time, server_id: rand::random::(), ..Default::default() @@ -96,9 +92,6 @@ impl ServerStats { /// Reports a server connection is no longer assigned to a client /// and is available for the next client to pick it up pub fn idle(&self) { - self.pool_stats - .server_idle(self.state.load(Ordering::Relaxed)); - self.state.store(ServerState::Idle, Ordering::Relaxed); } @@ -106,22 +99,16 @@ impl ServerStats { /// Also updates metrics on the pool regarding server usage. pub fn disconnect(&self) { self.reporter.server_disconnecting(self.server_id); - self.pool_stats - .server_disconnect(self.state.load(Ordering::Relaxed)) } /// Reports a server connection is being tested before being given to a client. pub fn tested(&self) { self.set_undefined_application(); - self.pool_stats - .server_tested(self.state.load(Ordering::Relaxed)); self.state.store(ServerState::Tested, Ordering::Relaxed); } /// Reports a server connection is attempting to login. pub fn login(&self) { - self.pool_stats - .server_login(self.state.load(Ordering::Relaxed)); self.state.store(ServerState::Login, Ordering::Relaxed); self.set_undefined_application(); } @@ -129,8 +116,6 @@ impl ServerStats { /// Reports a server connection has been assigned to a client that /// is about to query the server pub fn active(&self, application_name: String) { - self.pool_stats - .server_active(self.state.load(Ordering::Relaxed)); self.state.store(ServerState::Active, Ordering::Relaxed); self.set_application(application_name); } @@ -152,11 +137,11 @@ impl ServerStats { // Helper methods for show_servers pub fn pool_name(&self) -> String { - self.pool_stats.database() + self.address.pool_name.clone() } pub fn username(&self) -> String { - self.pool_stats.user() + self.address.username.clone() } pub fn address_name(&self) -> String { @@ -180,9 +165,6 @@ impl ServerStats { // Update server stats and address aggregation stats self.set_application(application_name); self.address.stats.wait_time_add(microseconds); - self.pool_stats - .maxwait - .fetch_max(microseconds, Ordering::Relaxed); } /// Report a query executed by a client against a server diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index f93b1a6c8..9a8523500 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -11,326 +11,6 @@ processes.pgcat.shutdown end - describe "SHOW STATS" do - context "clients connect and make one query" do - it "updates *_query_time and *_wait_time" do - connections = Array.new(3) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } - connections.each do |c| - Thread.new { c.async_exec("SELECT pg_sleep(0.25)") } - end - sleep(1) - connections.map(&:close) - - # wait for averages to be calculated, we shouldn't do this too often - sleep(15.5) - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - results = admin_conn.async_exec("SHOW STATS")[0] - admin_conn.close - expect(results["total_query_time"].to_i).to be_within(200).of(750) - expect(results["avg_query_time"].to_i).to be_within(50).of(250) - - expect(results["total_wait_time"].to_i).to_not eq(0) - expect(results["avg_wait_time"].to_i).to_not eq(0) - end - end - end - - describe "SHOW POOLS" do - context "bad credentials" do - it "does not change any stats" do - bad_password_url = URI(pgcat_conn_str) - bad_password_url.password = "wrong" - expect { PG::connect("#{bad_password_url.to_s}?application_name=bad_password") }.to raise_error(PG::ConnectionBad) - - sleep(1) - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - - expect(results["sv_idle"]).to eq("1") - end - end - - context "bad database name" do - it "does not change any stats" do - bad_db_url = URI(pgcat_conn_str) - bad_db_url.path = "/wrong_db" - expect { PG::connect("#{bad_db_url.to_s}?application_name=bad_db") }.to raise_error(PG::ConnectionBad) - - sleep(1) - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - - expect(results["sv_idle"]).to eq("1") - end - end - - context "client connects but issues no queries" do - it "only affects cl_idle stats" do - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - - before_test = admin_conn.async_exec("SHOW POOLS")[0]["sv_idle"] - connections = Array.new(20) { PG::connect(pgcat_conn_str) } - sleep(1) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - expect(results["cl_idle"]).to eq("20") - expect(results["sv_idle"]).to eq(before_test) - - connections.map(&:close) - sleep(1.1) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_active cl_idle cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - expect(results["sv_idle"]).to eq(before_test) - end - end - - context "clients connect and make one query" do - it "only affects cl_idle, sv_idle stats" do - connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } - connections.each do |c| - Thread.new { c.async_exec("SELECT pg_sleep(2.5)") } - end - - sleep(1.1) - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_idle cl_waiting cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - expect(results["cl_active"]).to eq("5") - expect(results["sv_active"]).to eq("5") - - sleep(3) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - expect(results["cl_idle"]).to eq("5") - expect(results["sv_idle"]).to eq("5") - - connections.map(&:close) - sleep(1) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - expect(results["sv_idle"]).to eq("5") - end - end - - context "client connects and opens a transaction and closes connection uncleanly" do - it "produces correct statistics" do - connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } - connections.each do |c| - Thread.new do - c.async_exec("BEGIN") - c.async_exec("SELECT pg_sleep(0.01)") - c.close - end - end - - sleep(1.1) - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - expect(results["sv_idle"]).to eq("5") - end - end - - context "client fail to checkout connection from the pool" do - it "counts clients as idle" do - new_configs = processes.pgcat.current_config - new_configs["general"]["connect_timeout"] = 500 - new_configs["general"]["ban_time"] = 1 - new_configs["general"]["shutdown_timeout"] = 1 - new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 - processes.pgcat.update_config(new_configs) - processes.pgcat.reload_config - - threads = [] - connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } - connections.each do |c| - threads << Thread.new { c.async_exec("SELECT pg_sleep(1)") rescue PG::SystemError } - end - - sleep(2) - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - expect(results["cl_idle"]).to eq("5") - expect(results["sv_idle"]).to eq("1") - - threads.map(&:join) - connections.map(&:close) - end - end - - context "clients connects and disconnect normally" do - let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) } - - it 'shows the same number of clients before and after' do - clients_before = clients_connected_to_pool(processes: processes) - threads = [] - connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } - connections.each do |c| - threads << Thread.new { c.async_exec("SELECT 1") } - end - clients_between = clients_connected_to_pool(processes: processes) - expect(clients_before).not_to eq(clients_between) - connections.each(&:close) - clients_after = clients_connected_to_pool(processes: processes) - expect(clients_before).to eq(clients_after) - end - end - - context "clients connects and disconnect abruptly" do - let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 10) } - - it 'shows the same number of clients before and after' do - threads = [] - connections = Array.new(2) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } - connections.each do |c| - threads << Thread.new { c.async_exec("SELECT 1") } - end - clients_before = clients_connected_to_pool(processes: processes) - random_string = (0...8).map { (65 + rand(26)).chr }.join - connection_string = "#{pgcat_conn_str}?application_name=#{random_string}" - faulty_client = Process.spawn("psql -Atx #{connection_string} >/dev/null") - sleep(1) - # psql starts two processes, we only know the pid of the parent, this - # ensure both are killed - `pkill -9 -f '#{random_string}'` - Process.wait(faulty_client) - clients_after = clients_connected_to_pool(processes: processes) - expect(clients_before).to eq(clients_after) - end - end - - context "clients overwhelm server pools" do - let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) } - - it "cl_waiting is updated to show it" do - threads = [] - connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } - connections.each do |c| - threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") } - end - - sleep(1.1) # Allow time for stats to update - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - - expect(results["cl_waiting"]).to eq("2") - expect(results["cl_active"]).to eq("2") - expect(results["sv_active"]).to eq("2") - - sleep(2.5) # Allow time for stats to update - results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login].each do |s| - raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" - end - expect(results["cl_idle"]).to eq("4") - expect(results["sv_idle"]).to eq("2") - - threads.map(&:join) - connections.map(&:close) - end - - it "show correct max_wait" do - threads = [] - connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } - connections.each do |c| - threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") } - end - - sleep(2.5) # Allow time for stats to update - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - results = admin_conn.async_exec("SHOW POOLS")[0] - - expect(results["maxwait"]).to eq("1") - expect(results["maxwait_us"].to_i).to be_within(200_000).of(500_000) - - sleep(4.5) # Allow time for stats to update - results = admin_conn.async_exec("SHOW POOLS")[0] - expect(results["maxwait"]).to eq("0") - - threads.map(&:join) - connections.map(&:close) - end - end - end - - describe "SHOW CLIENTS" do - it "reports correct number and application names" do - conn_str = processes.pgcat.connection_string("sharded_db", "sharding_user") - connections = Array.new(20) { |i| PG::connect("#{conn_str}?application_name=app#{i % 5}") } - - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - sleep(1) # Wait for stats to be updated - - results = admin_conn.async_exec("SHOW CLIENTS") - expect(results.count).to eq(21) # count admin clients - expect(results.select { |c| c["application_name"] == "app3" || c["application_name"] == "app4" }.count).to eq(8) - expect(results.select { |c| c["database"] == "pgcat" }.count).to eq(1) - - connections[0..5].map(&:close) - sleep(1) # Wait for stats to be updated - results = admin_conn.async_exec("SHOW CLIENTS") - expect(results.count).to eq(15) - - connections[6..].map(&:close) - sleep(1) # Wait for stats to be updated - expect(admin_conn.async_exec("SHOW CLIENTS").count).to eq(1) - admin_conn.close - end - - it "reports correct number of queries and transactions" do - conn_str = processes.pgcat.connection_string("sharded_db", "sharding_user") - - connections = Array.new(2) { |i| PG::connect("#{conn_str}?application_name=app#{i}") } - connections.each do |c| - c.async_exec("SELECT 1") - c.async_exec("SELECT 2") - c.async_exec("SELECT 3") - c.async_exec("BEGIN") - c.async_exec("SELECT 4") - c.async_exec("SELECT 5") - c.async_exec("COMMIT") - end - - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - sleep(1) # Wait for stats to be updated - - results = admin_conn.async_exec("SHOW CLIENTS") - expect(results.count).to eq(3) - normal_client_results = results.reject { |r| r["database"] == "pgcat" } - expect(normal_client_results[0]["transaction_count"]).to eq("4") - expect(normal_client_results[1]["transaction_count"]).to eq("4") - expect(normal_client_results[0]["query_count"]).to eq("7") - expect(normal_client_results[1]["query_count"]).to eq("7") - - admin_conn.close - connections.map(&:close) - end - end - describe "Manual Banning" do let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 10) } before do @@ -401,7 +81,7 @@ end end - describe "SHOW users" do + describe "SHOW USERS" do it "returns the right users" do admin_conn = PG::connect(processes.pgcat.admin_connection_string) results = admin_conn.async_exec("SHOW USERS")[0] diff --git a/tests/ruby/stats_spec.rb b/tests/ruby/stats_spec.rb new file mode 100644 index 000000000..132b80a9e --- /dev/null +++ b/tests/ruby/stats_spec.rb @@ -0,0 +1,369 @@ +# frozen_string_literal: true +require 'open3' +require_relative 'spec_helper' + +describe "Stats" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 10) } + let(:pgcat_conn_str) { processes.pgcat.connection_string("sharded_db", "sharding_user") } + + after do + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + + describe "SHOW STATS" do + context "clients connect and make one query" do + it "updates *_query_time and *_wait_time" do + connections = Array.new(3) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + Thread.new { c.async_exec("SELECT pg_sleep(0.25)") } + end + sleep(1) + connections.map(&:close) + + # wait for averages to be calculated, we shouldn't do this too often + sleep(15.5) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW STATS")[0] + admin_conn.close + expect(results["total_query_time"].to_i).to be_within(200).of(750) + expect(results["avg_query_time"].to_i).to be_within(50).of(250) + + expect(results["total_wait_time"].to_i).to_not eq(0) + expect(results["avg_wait_time"].to_i).to_not eq(0) + end + end + end + + describe "SHOW POOLS" do + context "bad credentials" do + it "does not change any stats" do + bad_password_url = URI(pgcat_conn_str) + bad_password_url.password = "wrong" + expect { PG::connect("#{bad_password_url.to_s}?application_name=bad_password") }.to raise_error(PG::ConnectionBad) + + sleep(1) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + + expect(results["sv_idle"]).to eq("1") + end + end + + context "bad database name" do + it "does not change any stats" do + bad_db_url = URI(pgcat_conn_str) + bad_db_url.path = "/wrong_db" + expect { PG::connect("#{bad_db_url.to_s}?application_name=bad_db") }.to raise_error(PG::ConnectionBad) + + sleep(1) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + + expect(results["sv_idle"]).to eq("1") + end + end + + context "client connects but issues no queries" do + it "only affects cl_idle stats" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + + before_test = admin_conn.async_exec("SHOW POOLS")[0]["sv_idle"] + connections = Array.new(20) { PG::connect(pgcat_conn_str) } + sleep(1) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_idle"]).to eq("20") + expect(results["sv_idle"]).to eq(before_test) + + connections.map(&:close) + sleep(1.1) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_idle cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["sv_idle"]).to eq(before_test) + end + end + + context "clients connect and make one query" do + it "only affects cl_idle, sv_idle stats" do + connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + Thread.new { c.async_exec("SELECT pg_sleep(2.5)") } + end + + sleep(1.1) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_waiting cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_active"]).to eq("5") + expect(results["sv_active"]).to eq("5") + + sleep(3) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_idle"]).to eq("5") + expect(results["sv_idle"]).to eq("5") + + connections.map(&:close) + sleep(1) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["sv_idle"]).to eq("5") + end + end + + context "client connects and opens a transaction and closes connection uncleanly" do + it "produces correct statistics" do + connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + Thread.new do + c.async_exec("BEGIN") + c.async_exec("SELECT pg_sleep(0.01)") + c.close + end + end + + sleep(1.1) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["sv_idle"]).to eq("5") + end + end + + context "client fail to checkout connection from the pool" do + it "counts clients as idle" do + new_configs = processes.pgcat.current_config + new_configs["general"]["connect_timeout"] = 500 + new_configs["general"]["ban_time"] = 1 + new_configs["general"]["shutdown_timeout"] = 1 + new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + + threads = [] + connections = Array.new(5) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT pg_sleep(1)") rescue PG::SystemError } + end + + sleep(2) + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_idle"]).to eq("5") + expect(results["sv_idle"]).to eq("1") + + threads.map(&:join) + connections.map(&:close) + end + end + + context "clients connects and disconnect normally" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) } + + it 'shows the same number of clients before and after' do + clients_before = clients_connected_to_pool(processes: processes) + threads = [] + connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT 1") rescue nil } + end + clients_between = clients_connected_to_pool(processes: processes) + expect(clients_before).not_to eq(clients_between) + connections.each(&:close) + clients_after = clients_connected_to_pool(processes: processes) + expect(clients_before).to eq(clients_after) + end + end + + context "clients connects and disconnect abruptly" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 10) } + + it 'shows the same number of clients before and after' do + threads = [] + connections = Array.new(2) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT 1") } + end + clients_before = clients_connected_to_pool(processes: processes) + random_string = (0...8).map { (65 + rand(26)).chr }.join + connection_string = "#{pgcat_conn_str}?application_name=#{random_string}" + faulty_client = Process.spawn("psql -Atx #{connection_string} >/dev/null") + sleep(1) + # psql starts two processes, we only know the pid of the parent, this + # ensure both are killed + `pkill -9 -f '#{random_string}'` + Process.wait(faulty_client) + clients_after = clients_connected_to_pool(processes: processes) + expect(clients_before).to eq(clients_after) + end + end + + context "clients overwhelm server pools" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) } + + it "cl_waiting is updated to show it" do + threads = [] + connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") } + end + + sleep(1.1) # Allow time for stats to update + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + + expect(results["cl_waiting"]).to eq("2") + expect(results["cl_active"]).to eq("2") + expect(results["sv_active"]).to eq("2") + + sleep(2.5) # Allow time for stats to update + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + expect(results["cl_idle"]).to eq("4") + expect(results["sv_idle"]).to eq("2") + + threads.map(&:join) + connections.map(&:close) + end + + it "show correct max_wait" do + threads = [] + connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") rescue nil } + end + + sleep(2.5) # Allow time for stats to update + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + + expect(results["maxwait"]).to eq("1") + expect(results["maxwait_us"].to_i).to be_within(200_000).of(500_000) + connections.map(&:close) + + sleep(4.5) # Allow time for stats to update + results = admin_conn.async_exec("SHOW POOLS")[0] + expect(results["maxwait"]).to eq("0") + + threads.map(&:join) + end + end + end + + describe "SHOW CLIENTS" do + it "reports correct number and application names" do + conn_str = processes.pgcat.connection_string("sharded_db", "sharding_user") + connections = Array.new(20) { |i| PG::connect("#{conn_str}?application_name=app#{i % 5}") } + + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + sleep(1) # Wait for stats to be updated + + results = admin_conn.async_exec("SHOW CLIENTS") + expect(results.count).to eq(21) # count admin clients + expect(results.select { |c| c["application_name"] == "app3" || c["application_name"] == "app4" }.count).to eq(8) + expect(results.select { |c| c["database"] == "pgcat" }.count).to eq(1) + + connections[0..5].map(&:close) + sleep(1) # Wait for stats to be updated + results = admin_conn.async_exec("SHOW CLIENTS") + expect(results.count).to eq(15) + + connections[6..].map(&:close) + sleep(1) # Wait for stats to be updated + expect(admin_conn.async_exec("SHOW CLIENTS").count).to eq(1) + admin_conn.close + end + + it "reports correct number of queries and transactions" do + conn_str = processes.pgcat.connection_string("sharded_db", "sharding_user") + + connections = Array.new(2) { |i| PG::connect("#{conn_str}?application_name=app#{i}") } + connections.each do |c| + c.async_exec("SELECT 1") + c.async_exec("SELECT 2") + c.async_exec("SELECT 3") + c.async_exec("BEGIN") + c.async_exec("SELECT 4") + c.async_exec("SELECT 5") + c.async_exec("COMMIT") + end + + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + sleep(1) # Wait for stats to be updated + + results = admin_conn.async_exec("SHOW CLIENTS") + expect(results.count).to eq(3) + normal_client_results = results.reject { |r| r["database"] == "pgcat" } + expect(normal_client_results[0]["transaction_count"]).to eq("4") + expect(normal_client_results[1]["transaction_count"]).to eq("4") + expect(normal_client_results[0]["query_count"]).to eq("7") + expect(normal_client_results[1]["query_count"]).to eq("7") + + admin_conn.close + connections.map(&:close) + end + end + + + describe "Query Storm" do + context "when the proxy receives overwhelmingly large number of short quick queries" do + it "should not have lingering clients or active servers" do + new_configs = processes.pgcat.current_config + + new_configs["general"]["connect_timeout"] = 500 + new_configs["general"]["ban_time"] = 1 + new_configs["general"]["shutdown_timeout"] = 1 + new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + + Array.new(40) do + Thread.new do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("SELECT pg_sleep(0.1)") + rescue PG::SystemError + ensure + conn.close + end + end.each(&:join) + + sleep 1 + + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW POOLS")[0] + %w[cl_idle cl_waiting cl_cancel_req sv_used sv_tested sv_login].each do |s| + raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" + end + + admin_conn.close + end + end + end +end From 211623f4530933e204a7757edc79e33ec34db964 Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Sun, 28 May 2023 22:46:13 -0400 Subject: [PATCH 412/551] Use latest bb8 and use Lifo as the queue strategy in the pool (#455) * Use git bb8 * Use latest bb8 and change pool is use stack --- Cargo.lock | 3 +-- Cargo.toml | 2 +- src/pool.rs | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d26c4d52b..c99453a92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,8 +84,7 @@ checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "bb8" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1627eccf3aa91405435ba240be23513eeca466b5dc33866422672264de061582" +source = "git+https://github.com/djc/bb8.git?rev=ad653e0#ad653e021607eb1f90ed6ce554d1766920308ffa" dependencies = [ "async-trait", "futures-channel", diff --git a/Cargo.toml b/Cargo.toml index 17f7e0e73..cee5683a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" tokio = { version = "1", features = ["full"] } bytes = "1" md-5 = "0.10" -bb8 = "0.8.0" +bb8 = { git = "https://github.com/djc/bb8.git", rev = "ad653e0" } # https://github.com/djc/bb8/commit/ad653e021607eb1f90ed6ce554d1766920308ffa async-trait = "0.1" rand = "0.8" chrono = "0.4" diff --git a/src/pool.rs b/src/pool.rs index 4fa277534..dff7b8e6f 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,6 +1,6 @@ use arc_swap::ArcSwap; use async_trait::async_trait; -use bb8::{ManageConnection, Pool, PooledConnection}; +use bb8::{ManageConnection, Pool, PooledConnection, QueueStrategy}; use bytes::{BufMut, BytesMut}; use chrono::naive::NaiveDateTime; use log::{debug, error, info, warn}; @@ -401,6 +401,7 @@ impl ConnectionPool { .idle_timeout(Some(std::time::Duration::from_millis(idle_timeout))) .max_lifetime(Some(std::time::Duration::from_millis(server_lifetime))) .reaper_rate(std::time::Duration::from_millis(reaper_rate)) + .queue_strategy(QueueStrategy::Lifo) .test_on_check_out(false); let pool = if config.general.validate_config { From cc1dce36f42f6109b5340425acc478b93f513b35 Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 31 May 2023 11:25:42 -0400 Subject: [PATCH 413/551] Change default server lifetime and bump bb8 version to use LIFO correctly (#462) Change default server lifetime and idle timeouts and bump bb8 version to use LIFO correctly --- Cargo.lock | 5 +++-- Cargo.toml | 2 +- src/config.rs | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c99453a92..95fd682d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,8 +83,9 @@ checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "bb8" -version = "0.8.0" -source = "git+https://github.com/djc/bb8.git?rev=ad653e0#ad653e021607eb1f90ed6ce554d1766920308ffa" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b4b0f25f18bcdc3ac72bdb486ed0acf7e185221fd4dc985bc15db5800b0ba2" dependencies = [ "async-trait", "futures-channel", diff --git a/Cargo.toml b/Cargo.toml index cee5683a8..e26323596 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" tokio = { version = "1", features = ["full"] } bytes = "1" md-5 = "0.10" -bb8 = { git = "https://github.com/djc/bb8.git", rev = "ad653e0" } # https://github.com/djc/bb8/commit/ad653e021607eb1f90ed6ce554d1766920308ffa +bb8 = "0.8.1" async-trait = "0.1" rand = "0.8" chrono = "0.4" diff --git a/src/config.rs b/src/config.rs index 2f5596c60..4468f7393 100644 --- a/src/config.rs +++ b/src/config.rs @@ -329,7 +329,7 @@ impl General { } pub fn default_server_lifetime() -> u64 { - 1000 * 60 * 60 * 24 // 24 hours + 1000 * 60 * 60 // 1 hour } pub fn default_connect_timeout() -> u64 { @@ -352,7 +352,7 @@ impl General { } pub fn default_idle_timeout() -> u64 { - 60000 // 10 minutes + 60000 // 1 minute } pub fn default_shutdown_timeout() -> u64 { From 1e86104501f8544f1ca3365bd9171cb104016a9a Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Fri, 9 Jun 2023 14:35:20 -0400 Subject: [PATCH 414/551] Make queue strategy configurable and default to Fifo (#463) * Change idle timeout default to 10 minutes * Revert lifo for now while we investigate connection thrashing issues * Make queue strategy configurable * test revert idle time out * Add pgcat start to python test --- src/config.rs | 13 +++++++++++-- src/pool.rs | 7 ++++++- tests/python/tests.py | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/config.rs b/src/config.rs index 4468f7393..7573d39b8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -292,6 +292,9 @@ pub struct General { #[serde(default = "General::default_server_lifetime")] pub server_lifetime: u64, + #[serde(default = "General::default_server_round_robin")] // False + pub server_round_robin: bool, + #[serde(default = "General::default_worker_threads")] pub worker_threads: usize, @@ -352,7 +355,7 @@ impl General { } pub fn default_idle_timeout() -> u64 { - 60000 // 1 minute + 600000 // 10 minutes } pub fn default_shutdown_timeout() -> u64 { @@ -390,6 +393,10 @@ impl General { pub fn default_prometheus_exporter_port() -> i16 { 9930 } + + pub fn default_server_round_robin() -> bool { + true + } } impl Default for General { @@ -424,7 +431,8 @@ impl Default for General { auth_query: None, auth_query_user: None, auth_query_password: None, - server_lifetime: 1000 * 3600 * 24, // 24 hours, + server_lifetime: Self::default_server_lifetime(), + server_round_robin: false, validate_config: true, } } @@ -983,6 +991,7 @@ impl Config { "Default max server lifetime: {}ms", self.general.server_lifetime ); + info!("Sever round robin: {}", self.general.server_round_robin); match self.general.tls_certificate.clone() { Some(tls_certificate) => { info!("TLS certificate: {}", tls_certificate); diff --git a/src/pool.rs b/src/pool.rs index dff7b8e6f..b92935217 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -389,6 +389,11 @@ impl ConnectionPool { .min() .unwrap(); + let queue_strategy = match config.general.server_round_robin { + true => QueueStrategy::Fifo, + false => QueueStrategy::Lifo, + }; + debug!( "[pool: {}][user: {}] Pool reaper rate: {}ms", pool_name, user.username, reaper_rate @@ -401,7 +406,7 @@ impl ConnectionPool { .idle_timeout(Some(std::time::Duration::from_millis(idle_timeout))) .max_lifetime(Some(std::time::Duration::from_millis(server_lifetime))) .reaper_rate(std::time::Duration::from_millis(reaper_rate)) - .queue_strategy(QueueStrategy::Lifo) + .queue_strategy(queue_strategy) .test_on_check_out(false); let pool = if config.general.validate_config { diff --git a/tests/python/tests.py b/tests/python/tests.py index 6108ff218..cd54081f4 100644 --- a/tests/python/tests.py +++ b/tests/python/tests.py @@ -63,6 +63,7 @@ def cleanup_conn(conn: psycopg2.extensions.connection, cur: psycopg2.extensions. def test_normal_db_access(): + pgcat_start() conn, cur = connect_db(autocommit=False) cur.execute("SELECT 1") res = cur.fetchall() From 373c6f14cf7c99c5588ab6ebfe99b3f96cb9735a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 00:50:26 -0700 Subject: [PATCH 415/551] chore(deps): bump log from 0.4.17 to 0.4.19 (#470) Bumps [log](https://github.com/rust-lang/log) from 0.4.17 to 0.4.19. - [Release notes](https://github.com/rust-lang/log/releases) - [Changelog](https://github.com/rust-lang/log/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/log/compare/0.4.17...0.4.19) --- updated-dependencies: - dependency-name: log dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95fd682d1..f4329dfc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -753,12 +753,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "lru-cache" From 8a8f034e61c4b9471478d031a4a9e40ff44e7358 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 00:50:46 -0700 Subject: [PATCH 416/551] chore(deps): bump regex from 1.8.1 to 1.8.4 (#466) Bumps [regex](https://github.com/rust-lang/regex) from 1.8.1 to 1.8.4. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.8.1...1.8.4) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4329dfc4..f2b9a440b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1103,9 +1103,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.1" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", @@ -1114,9 +1114,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "resolv-conf" From 159badf21dba4db11407cf51141c59207f301101 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 00:51:05 -0700 Subject: [PATCH 417/551] chore(deps): bump sqlparser from 0.33.0 to 0.34.0 (#448) Bumps [sqlparser](https://github.com/sqlparser-rs/sqlparser-rs) from 0.33.0 to 0.34.0. - [Changelog](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/sqlparser-rs/sqlparser-rs/compare/v0.33.0...v0.34.0) --- updated-dependencies: - dependency-name: sqlparser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2b9a440b..51ba9652c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1326,9 +1326,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355dc4d4b6207ca8a3434fc587db0a8016130a574dbcdbfb93d7f7b5bc5b211a" +checksum = "37d3706eefb17039056234df6b566b0014f303f867f2656108334a55b8096f59" dependencies = [ "log", "sqlparser_derive", diff --git a/Cargo.toml b/Cargo.toml index e26323596..a70c7e938 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = {version = "0.33", features = ["visitor"] } +sqlparser = {version = "0.34", features = ["visitor"] } log = "0.4" arc-swap = "1" env_logger = "0.10" From 60b898ba1ca4b8daa19bc00c12a9790407dad843 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 00:51:24 -0700 Subject: [PATCH 418/551] chore(deps): bump pin-project from 1.0.12 to 1.1.0 (#440) Bumps [pin-project](https://github.com/taiki-e/pin-project) from 1.0.12 to 1.1.0. - [Release notes](https://github.com/taiki-e/pin-project/releases) - [Changelog](https://github.com/taiki-e/pin-project/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/pin-project/compare/v1.0.12...v1.1.0) --- updated-dependencies: - dependency-name: pin-project dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51ba9652c..0baef1ced 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -984,22 +984,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.9", ] [[package]] From 70eaf9d72a0986889408eca9a3ed1f181f69f7da Mon Sep 17 00:00:00 2001 From: Jeff Chen Date: Mon, 12 Jun 2023 10:23:56 -0600 Subject: [PATCH 419/551] Report min_pool_size correctly (#471) --- src/admin.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 804476c90..9ae5e9d58 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -308,17 +308,17 @@ where let paused = pool.paused(); res.put(data_row(&vec![ - address.name(), // name - address.host.to_string(), // host - address.port.to_string(), // port - database_name.to_string(), // database - pool_config.user.username.to_string(), // force_user - pool_config.user.pool_size.to_string(), // pool_size - "0".to_string(), // min_pool_size - "0".to_string(), // reserve_pool - pool_config.pool_mode.to_string(), // pool_mode - pool_config.user.pool_size.to_string(), // max_connections - pool_state.connections.to_string(), // current_connections + address.name(), // name + address.host.to_string(), // host + address.port.to_string(), // port + database_name.to_string(), // database + pool_config.user.username.to_string(), // force_user + pool_config.user.pool_size.to_string(), // pool_size + pool_config.user.min_pool_size.unwrap_or(0).to_string(), // min_pool_size + "0".to_string(), // reserve_pool + pool_config.pool_mode.to_string(), // pool_mode + pool_config.user.pool_size.to_string(), // max_connections + pool_state.connections.to_string(), // current_connections match paused { // paused true => "1".to_string(), From 9b58049036cc2843cf9dbaaae136be21a91b7650 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 16 Jun 2023 12:57:44 -0700 Subject: [PATCH 420/551] Support for prepared statements (#474) * Start prepared statements * parse * Ok * optional * dont rewrite anonymous prepared stmts * Dont rewrite anonymous prep statements * hm? * prep statements * I see! * comment * Print config value * Rewrite bind and add sqlx test * fmt * ok * Fix * Fix stats * its late * clean up PREPARE --- dev/Dockerfile | 2 +- pgcat.toml | 3 + src/admin.rs | 10 + src/client.rs | 215 ++++++- src/config.rs | 9 + src/errors.rs | 7 + src/messages.rs | 293 +++++++++ src/query_router.rs | 5 +- src/server.rs | 43 +- src/stats/server.rs | 15 + tests/rust/.gitignore | 1 + tests/rust/Cargo.lock | 1322 ++++++++++++++++++++++++++++++++++++++++ tests/rust/Cargo.toml | 10 + tests/rust/src/main.rs | 29 + 14 files changed, 1954 insertions(+), 10 deletions(-) create mode 100644 tests/rust/.gitignore create mode 100644 tests/rust/Cargo.lock create mode 100644 tests/rust/Cargo.toml create mode 100644 tests/rust/src/main.rs diff --git a/dev/Dockerfile b/dev/Dockerfile index f2054ef2d..bd24174f8 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:bullseye +FROM rust:1.70-bullseye # Dependencies RUN apt-get update -y \ diff --git a/pgcat.toml b/pgcat.toml index e6b54b25d..41d0210a0 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -60,6 +60,9 @@ tcp_keepalives_count = 5 # Number of seconds between keepalive packets. tcp_keepalives_interval = 5 +# Handle prepared statements. +prepared_statements = true + # Path to TLS Certificate file to use for TLS connections # tls_certificate = ".circleci/server.cert" # Path to TLS private key file to use for TLS connections diff --git a/src/admin.rs b/src/admin.rs index 9ae5e9d58..bbca956f9 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -699,6 +699,8 @@ where ("bytes_sent", DataType::Numeric), ("bytes_received", DataType::Numeric), ("age_seconds", DataType::Numeric), + ("prepare_cache_hit", DataType::Numeric), + ("prepare_cache_miss", DataType::Numeric), ]; let new_map = get_server_stats(); @@ -722,6 +724,14 @@ where .duration_since(server.connect_time()) .as_secs() .to_string(), + server + .prepared_hit_count + .load(Ordering::Relaxed) + .to_string(), + server + .prepared_miss_count + .load(Ordering::Relaxed) + .to_string(), ]; res.put(data_row(&row)); diff --git a/src/client.rs b/src/client.rs index 1ff558b5a..608d838df 100644 --- a/src/client.rs +++ b/src/client.rs @@ -3,8 +3,9 @@ use crate::pool::BanReason; /// Handle clients by pretending to be a PostgreSQL server. use bytes::{Buf, BufMut, BytesMut}; use log::{debug, error, info, trace, warn}; +use once_cell::sync::Lazy; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{atomic::AtomicUsize, Arc}; use std::time::Instant; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; use tokio::net::TcpStream; @@ -13,7 +14,9 @@ use tokio::sync::mpsc::Sender; use crate::admin::{generate_server_info_for_admin, handle_admin}; use crate::auth_passthrough::refetch_auth_hash; -use crate::config::{get_config, get_idle_client_in_transaction_timeout, Address, PoolMode}; +use crate::config::{ + get_config, get_idle_client_in_transaction_timeout, get_prepared_statements, Address, PoolMode, +}; use crate::constants::*; use crate::messages::*; use crate::plugins::PluginOutput; @@ -25,6 +28,11 @@ use crate::tls::Tls; use tokio_rustls::server::TlsStream; +/// Incrementally count prepared statements +/// to avoid random conflicts in places where the random number generator is weak. +pub static PREPARED_STATEMENT_COUNTER: Lazy> = + Lazy::new(|| Arc::new(AtomicUsize::new(0))); + /// Type of connection received from client. enum ClientConnectionType { Startup, @@ -93,6 +101,9 @@ pub struct Client { /// Used to notify clients about an impending shutdown shutdown: Receiver<()>, + + /// Prepared statements + prepared_statements: HashMap, } /// Client entrypoint. @@ -682,6 +693,7 @@ where application_name: application_name.to_string(), shutdown, connected_to_server: false, + prepared_statements: HashMap::new(), }) } @@ -716,6 +728,7 @@ where application_name: String::from("undefined"), shutdown, connected_to_server: false, + prepared_statements: HashMap::new(), }) } @@ -757,6 +770,10 @@ where // Result returned by one of the plugins. let mut plugin_output = None; + // Prepared statement being executed + let mut prepared_statement = None; + let mut will_prepare = false; + // Our custom protocol loop. // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocol. @@ -766,13 +783,16 @@ where self.transaction_mode ); + // Should we rewrite prepared statements and bind messages? + let mut prepared_statements_enabled = get_prepared_statements(); + // Read a complete message from the client, which normally would be // either a `Q` (query) or `P` (prepare, extended protocol). // We can parse it here before grabbing a server from the pool, // in case the client is sending some custom protocol messages, e.g. // SET SHARDING KEY TO 'bigint'; - let message = tokio::select! { + let mut message = tokio::select! { _ = self.shutdown.recv() => { if !self.admin { error_response_terminal( @@ -800,7 +820,21 @@ where // allocate a connection, we wouldn't be able to send back an error message // to the client so we buffer them and defer the decision to error out or not // to when we get the S message - 'D' | 'E' => { + 'D' => { + if prepared_statements_enabled { + let name; + (name, message) = self.rewrite_describe(message).await?; + + if let Some(name) = name { + prepared_statement = Some(name); + } + } + + self.buffer.put(&message[..]); + continue; + } + + 'E' => { self.buffer.put(&message[..]); continue; } @@ -830,6 +864,11 @@ where } 'P' => { + if prepared_statements_enabled { + (prepared_statement, message) = self.rewrite_parse(message)?; + will_prepare = true; + } + self.buffer.put(&message[..]); if query_router.query_parser_enabled() { @@ -846,6 +885,10 @@ where } 'B' => { + if prepared_statements_enabled { + (prepared_statement, message) = self.rewrite_bind(message).await?; + } + self.buffer.put(&message[..]); if query_router.query_parser_enabled() { @@ -1054,7 +1097,48 @@ where // If the client is in session mode, no more custom protocol // commands will be accepted. loop { - let message = match initial_message { + // Only check if we should rewrite prepared statements + // in session mode. In transaction mode, we check at the beginning of + // each transaction. + if !self.transaction_mode { + prepared_statements_enabled = get_prepared_statements(); + } + + debug!("Prepared statement active: {:?}", prepared_statement); + + // We are processing a prepared statement. + if let Some(ref name) = prepared_statement { + debug!("Checking prepared statement is on server"); + // Get the prepared statement the server expects to see. + let statement = match self.prepared_statements.get(name) { + Some(statement) => { + debug!("Prepared statement `{}` found in cache", name); + statement + } + None => { + return Err(Error::ClientError(format!( + "prepared statement `{}` not found", + name + ))) + } + }; + + // Since it's already in the buffer, we don't need to prepare it on this server. + if will_prepare { + server.will_prepare(&statement.name); + will_prepare = false; + } else { + // The statement is not prepared on the server, so we need to prepare it. + if server.should_prepare(&statement.name) { + server.prepare(statement).await?; + } + } + + // Done processing the prepared statement. + prepared_statement = None; + } + + let mut message = match initial_message { None => { trace!("Waiting for message inside transaction or in session mode"); @@ -1173,6 +1257,11 @@ where // Parse // The query with placeholders is here, e.g. `SELECT * FROM users WHERE email = $1 AND active = $2`. 'P' => { + if prepared_statements_enabled { + (prepared_statement, message) = self.rewrite_parse(message)?; + will_prepare = true; + } + if query_router.query_parser_enabled() { if let Ok(ast) = QueryRouter::parse(&message) { if let Ok(output) = query_router.execute_plugins(&ast).await { @@ -1187,12 +1276,25 @@ where // Bind // The placeholder's replacements are here, e.g. 'user@email.com' and 'true' 'B' => { + if prepared_statements_enabled { + (prepared_statement, message) = self.rewrite_bind(message).await?; + } + self.buffer.put(&message[..]); } // Describe // Command a client can issue to describe a previously prepared named statement. 'D' => { + if prepared_statements_enabled { + let name; + (name, message) = self.rewrite_describe(message).await?; + + if let Some(name) = name { + prepared_statement = Some(name); + } + } + self.buffer.put(&message[..]); } @@ -1235,7 +1337,7 @@ where let first_message_code = (*self.buffer.get(0).unwrap_or(&0)) as char; // Almost certainly true - if first_message_code == 'P' { + if first_message_code == 'P' && !prepared_statements_enabled { // Message layout // P followed by 32 int followed by null-terminated statement name // So message code should be in offset 0 of the buffer, first character @@ -1363,6 +1465,107 @@ where } } + /// Rewrite Parse (F) message to set the prepared statement name to one we control. + /// Save it into the client cache. + fn rewrite_parse(&mut self, message: BytesMut) -> Result<(Option, BytesMut), Error> { + let parse: Parse = (&message).try_into()?; + + let name = parse.name.clone(); + + // Don't rewrite anonymous prepared statements + if parse.anonymous() { + debug!("Anonymous prepared statement"); + return Ok((None, message)); + } + + let parse = parse.rename(); + + debug!( + "Renamed prepared statement `{}` to `{}` and saved to cache", + name, parse.name + ); + + self.prepared_statements.insert(name.clone(), parse.clone()); + + Ok((Some(name), parse.try_into()?)) + } + + /// Rewrite the Bind (F) message to use the prepared statement name + /// saved in the client cache. + async fn rewrite_bind( + &mut self, + message: BytesMut, + ) -> Result<(Option, BytesMut), Error> { + let bind: Bind = (&message).try_into()?; + let name = bind.prepared_statement.clone(); + + if bind.anonymous() { + debug!("Anonymous bind message"); + return Ok((None, message)); + } + + match self.prepared_statements.get(&name) { + Some(prepared_stmt) => { + let bind = bind.reassign(prepared_stmt); + + debug!("Rewrote bind `{}` to `{}`", name, bind.prepared_statement); + + Ok((Some(name), bind.try_into()?)) + } + None => { + debug!("Got bind for unknown prepared statement {:?}", bind); + + error_response( + &mut self.write, + &format!( + "prepared statement \"{}\" does not exist", + bind.prepared_statement + ), + ) + .await?; + + Err(Error::ClientError(format!( + "Prepared statement `{}` doesn't exist", + name + ))) + } + } + } + + /// Rewrite the Describe (F) message to use the prepared statement name + /// saved in the client cache. + async fn rewrite_describe( + &mut self, + message: BytesMut, + ) -> Result<(Option, BytesMut), Error> { + let describe: Describe = (&message).try_into()?; + let name = describe.statement_name.clone(); + + if describe.anonymous() { + debug!("Anonymous describe"); + return Ok((None, message)); + } + + match self.prepared_statements.get(&name) { + Some(prepared_stmt) => { + let describe = describe.rename(&prepared_stmt.name); + + debug!( + "Rewrote describe `{}` to `{}`", + name, describe.statement_name + ); + + Ok((Some(name), describe.try_into()?)) + } + + None => { + debug!("Got describe for unknown prepared statement {:?}", describe); + + Ok((None, message)) + } + } + } + /// Release the server from the client: it can't cancel its queries anymore. pub fn release(&self) { let mut guard = self.client_server_map.lock(); diff --git a/src/config.rs b/src/config.rs index 7573d39b8..66c20758e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -320,6 +320,9 @@ pub struct General { pub auth_query: Option, pub auth_query_user: Option, pub auth_query_password: Option, + + #[serde(default)] + pub prepared_statements: bool, } impl General { @@ -434,6 +437,7 @@ impl Default for General { server_lifetime: Self::default_server_lifetime(), server_round_robin: false, validate_config: true, + prepared_statements: false, } } } @@ -1015,6 +1019,7 @@ impl Config { "Server TLS certificate verification: {}", self.general.verify_server_certificate ); + info!("Prepared statements: {}", self.general.prepared_statements); info!( "Plugins: {}", match self.plugins { @@ -1239,6 +1244,10 @@ pub fn get_idle_client_in_transaction_timeout() -> u64 { .idle_client_in_transaction_timeout } +pub fn get_prepared_statements() -> bool { + (*(*CONFIG.load())).general.prepared_statements +} + /// Parse the configuration file located at the path. pub async fn parse(path: &str) -> Result<(), Error> { let mut contents = String::new(); diff --git a/src/errors.rs b/src/errors.rs index b1796eee2..c076a3f93 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -26,6 +26,7 @@ pub enum Error { AuthPassthroughError(String), UnsupportedStatement, QueryRouterParserError(String), + QueryRouterError(String), } #[derive(Clone, PartialEq, Debug)] @@ -121,3 +122,9 @@ impl std::fmt::Display for Error { } } } + +impl From for Error { + fn from(err: std::ffi::NulError) -> Self { + Error::QueryRouterError(err.to_string()) + } +} diff --git a/src/messages.rs b/src/messages.rs index ee4886df2..552497f15 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -7,11 +7,15 @@ use socket2::{SockRef, TcpKeepalive}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; +use crate::client::PREPARED_STATEMENT_COUNTER; use crate::config::get_config; use crate::errors::Error; + use std::collections::HashMap; +use std::ffi::CString; use std::io::{BufRead, Cursor}; use std::mem; +use std::sync::atomic::Ordering; use std::time::Duration; /// Postgres data type mappings @@ -526,6 +530,13 @@ pub fn command_complete(command: &str) -> BytesMut { res } +pub fn flush() -> BytesMut { + let mut bytes = BytesMut::new(); + bytes.put_u8(b'H'); + bytes.put_i32(4); + bytes +} + /// Write all data in the buffer to the TcpStream. pub async fn write_all(stream: &mut S, buf: BytesMut) -> Result<(), Error> where @@ -689,3 +700,285 @@ impl BytesMutReader for Cursor<&BytesMut> { } } } + +/// Parse (F) message. +/// See: +#[derive(Clone, Debug)] +pub struct Parse { + code: char, + #[allow(dead_code)] + len: i32, + pub name: String, + pub generated_name: String, + query: String, + num_params: i16, + param_types: Vec, +} + +impl TryFrom<&BytesMut> for Parse { + type Error = Error; + + fn try_from(buf: &BytesMut) -> Result { + let mut cursor = Cursor::new(buf); + let code = cursor.get_u8() as char; + let len = cursor.get_i32(); + let name = cursor.read_string()?; + let query = cursor.read_string()?; + let num_params = cursor.get_i16(); + let mut param_types = Vec::new(); + + for _ in 0..num_params { + param_types.push(cursor.get_i32()); + } + + Ok(Parse { + code, + len, + name, + generated_name: prepared_statement_name(), + query, + num_params, + param_types, + }) + } +} + +impl TryFrom for BytesMut { + type Error = Error; + + fn try_from(parse: Parse) -> Result { + let mut bytes = BytesMut::new(); + + let name_binding = CString::new(parse.name)?; + let name = name_binding.as_bytes_with_nul(); + + let query_binding = CString::new(parse.query)?; + let query = query_binding.as_bytes_with_nul(); + + // Recompute length of the message. + let len = 4 // self + + name.len() + + query.len() + + 2 + + 4 * parse.num_params as usize; + + bytes.put_u8(parse.code as u8); + bytes.put_i32(len as i32); + bytes.put_slice(name); + bytes.put_slice(query); + bytes.put_i16(parse.num_params); + for param in parse.param_types { + bytes.put_i32(param); + } + + Ok(bytes) + } +} + +impl TryFrom<&Parse> for BytesMut { + type Error = Error; + + fn try_from(parse: &Parse) -> Result { + parse.clone().try_into() + } +} + +impl Parse { + pub fn rename(mut self) -> Self { + self.name = self.generated_name.to_string(); + self + } + + pub fn anonymous(&self) -> bool { + self.name.is_empty() + } +} + +/// Bind (B) message. +/// See: +#[derive(Clone, Debug)] +pub struct Bind { + code: char, + #[allow(dead_code)] + len: i64, + portal: String, + pub prepared_statement: String, + num_param_format_codes: i16, + param_format_codes: Vec, + num_param_values: i16, + param_values: Vec<(i32, BytesMut)>, + num_result_column_format_codes: i16, + result_columns_format_codes: Vec, +} + +impl TryFrom<&BytesMut> for Bind { + type Error = Error; + + fn try_from(buf: &BytesMut) -> Result { + let mut cursor = Cursor::new(buf); + let code = cursor.get_u8() as char; + let len = cursor.get_i32(); + let portal = cursor.read_string()?; + let prepared_statement = cursor.read_string()?; + let num_param_format_codes = cursor.get_i16(); + let mut param_format_codes = Vec::new(); + + for _ in 0..num_param_format_codes { + param_format_codes.push(cursor.get_i16()); + } + + let num_param_values = cursor.get_i16(); + let mut param_values = Vec::new(); + + for _ in 0..num_param_values { + let param_len = cursor.get_i32(); + let mut param = BytesMut::with_capacity(param_len as usize); + param.resize(param_len as usize, b'0'); + cursor.copy_to_slice(&mut param); + param_values.push((param_len, param)); + } + + let num_result_column_format_codes = cursor.get_i16(); + let mut result_columns_format_codes = Vec::new(); + + for _ in 0..num_result_column_format_codes { + result_columns_format_codes.push(cursor.get_i16()); + } + + Ok(Bind { + code, + len: len as i64, + portal, + prepared_statement, + num_param_format_codes, + param_format_codes, + num_param_values, + param_values, + num_result_column_format_codes, + result_columns_format_codes, + }) + } +} + +impl TryFrom for BytesMut { + type Error = Error; + + fn try_from(bind: Bind) -> Result { + let mut bytes = BytesMut::new(); + + let portal_binding = CString::new(bind.portal)?; + let portal = portal_binding.as_bytes_with_nul(); + + let prepared_statement_binding = CString::new(bind.prepared_statement)?; + let prepared_statement = prepared_statement_binding.as_bytes_with_nul(); + + let mut len = 4 // self + + portal.len() + + prepared_statement.len() + + 2 // num_param_format_codes + + 2 * bind.num_param_format_codes as usize // num_param_format_codes + + 2; // num_param_values + + for (param_len, _) in &bind.param_values { + len += 4 + *param_len as usize; + } + len += 2; // num_result_column_format_codes + len += 2 * bind.num_result_column_format_codes as usize; + + bytes.put_u8(bind.code as u8); + bytes.put_i32(len as i32); + bytes.put_slice(portal); + bytes.put_slice(prepared_statement); + bytes.put_i16(bind.num_param_format_codes); + for param_format_code in bind.param_format_codes { + bytes.put_i16(param_format_code); + } + bytes.put_i16(bind.num_param_values); + for (param_len, param) in bind.param_values { + bytes.put_i32(param_len); + bytes.put_slice(¶m); + } + bytes.put_i16(bind.num_result_column_format_codes); + for result_column_format_code in bind.result_columns_format_codes { + bytes.put_i16(result_column_format_code); + } + + Ok(bytes) + } +} + +impl Bind { + pub fn reassign(mut self, parse: &Parse) -> Self { + self.prepared_statement = parse.name.clone(); + self + } + + pub fn anonymous(&self) -> bool { + self.prepared_statement.is_empty() + } +} + +#[derive(Debug, Clone)] +pub struct Describe { + code: char, + + #[allow(dead_code)] + len: i32, + target: char, + pub statement_name: String, +} + +impl TryFrom<&BytesMut> for Describe { + type Error = Error; + + fn try_from(bytes: &BytesMut) -> Result { + let mut cursor = Cursor::new(bytes); + let code = cursor.get_u8() as char; + let len = cursor.get_i32(); + let target = cursor.get_u8() as char; + let statement_name = cursor.read_string()?; + + Ok(Describe { + code, + len, + target, + statement_name, + }) + } +} + +impl TryFrom for BytesMut { + type Error = Error; + + fn try_from(describe: Describe) -> Result { + let mut bytes = BytesMut::new(); + let statement_name_binding = CString::new(describe.statement_name)?; + let statement_name = statement_name_binding.as_bytes_with_nul(); + let len = 4 + 1 + statement_name.len(); + + bytes.put_u8(describe.code as u8); + bytes.put_i32(len as i32); + bytes.put_u8(describe.target as u8); + bytes.put_slice(statement_name); + + Ok(bytes) + } +} + +impl Describe { + pub fn rename(mut self, name: &str) -> Self { + self.statement_name = name.to_string(); + self + } + + pub fn anonymous(&self) -> bool { + self.statement_name.is_empty() + } +} + +pub fn prepared_statement_name() -> String { + format!( + "P_{}", + PREPARED_STATEMENT_COUNTER.fetch_add(1, Ordering::SeqCst) + ) +} diff --git a/src/query_router.rs b/src/query_router.rs index 3e3a23a8f..126b81380 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -331,7 +331,7 @@ impl QueryRouter { Some((command, value)) } - pub fn parse(message: &BytesMut) -> Result, Error> { + pub fn parse(message: &BytesMut) -> Result, Error> { let mut message_cursor = Cursor::new(message); let code = message_cursor.get_u8() as char; @@ -348,12 +348,13 @@ impl QueryRouter { // Parse (prepared statement) 'P' => { // Reads statement name - message_cursor.read_string().unwrap(); + let _name = message_cursor.read_string().unwrap(); // Reads query string let query = message_cursor.read_string().unwrap(); debug!("Prepared statement: '{}'", query); + query } diff --git a/src/server.rs b/src/server.rs index 32dd91f8b..ab29db09a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -5,7 +5,7 @@ use fallible_iterator::FallibleIterator; use log::{debug, error, info, trace, warn}; use parking_lot::{Mutex, RwLock}; use postgres_protocol::message; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::io::Read; use std::net::IpAddr; use std::sync::Arc; @@ -198,6 +198,9 @@ pub struct Server { /// Should clean up dirty connections? cleanup_connections: bool, + + /// Prepared statements + prepared_statements: BTreeSet, } impl Server { @@ -692,6 +695,7 @@ impl Server { )), }, cleanup_connections, + prepared_statements: BTreeSet::new(), }; server.set_name("pgcat").await?; @@ -910,6 +914,43 @@ impl Server { Ok(bytes) } + pub fn will_prepare(&mut self, name: &str) { + debug!("Will prepare `{}`", name); + + self.prepared_statements.insert(name.to_string()); + } + + pub fn should_prepare(&self, name: &str) -> bool { + let should_prepare = !self.prepared_statements.contains(name); + + debug!("Should prepare `{}`: {}", name, should_prepare); + + if should_prepare { + self.stats.prepared_cache_miss(); + } else { + self.stats.prepared_cache_hit(); + } + + should_prepare + } + + pub async fn prepare(&mut self, parse: &Parse) -> Result<(), Error> { + debug!("Preparing `{}`", parse.name); + + let bytes: BytesMut = parse.try_into()?; + self.send(&bytes).await?; + self.send(&flush()).await?; + + // Read and discard ParseComplete (B) + let _ = read_message(&mut self.stream).await?; + + self.prepared_statements.insert(parse.name.to_string()); + + debug!("Prepared `{}`", parse.name); + + Ok(()) + } + /// If the server is still inside a transaction. /// If the client disconnects while the server is in a transaction, we will clean it up. pub fn in_transaction(&self) -> bool { diff --git a/src/stats/server.rs b/src/stats/server.rs index e156ee094..6fb2dc973 100644 --- a/src/stats/server.rs +++ b/src/stats/server.rs @@ -47,6 +47,8 @@ pub struct ServerStats { pub transaction_count: Arc, pub query_count: Arc, pub error_count: Arc, + pub prepared_hit_count: Arc, + pub prepared_miss_count: Arc, } impl Default for ServerStats { @@ -63,6 +65,8 @@ impl Default for ServerStats { query_count: Arc::new(AtomicU64::new(0)), error_count: Arc::new(AtomicU64::new(0)), reporter: get_reporter(), + prepared_hit_count: Arc::new(AtomicU64::new(0)), + prepared_miss_count: Arc::new(AtomicU64::new(0)), } } } @@ -172,6 +176,7 @@ impl ServerStats { self.set_application(application_name.to_string()); self.address.stats.query_count_add(); self.address.stats.query_time_add(milliseconds); + self.query_count.fetch_add(1, Ordering::Relaxed); } /// Report a transaction executed by a client a server @@ -198,4 +203,14 @@ impl ServerStats { .fetch_add(amount_bytes as u64, Ordering::Relaxed); self.address.stats.bytes_received_add(amount_bytes as u64); } + + /// Report a prepared statement that already exists on the server. + pub fn prepared_cache_hit(&self) { + self.prepared_hit_count.fetch_add(1, Ordering::Relaxed); + } + + /// Report a prepared statement that does not exist on the server yet. + pub fn prepared_cache_miss(&self) { + self.prepared_miss_count.fetch_add(1, Ordering::Relaxed); + } } diff --git a/tests/rust/.gitignore b/tests/rust/.gitignore new file mode 100644 index 000000000..2f7896d1d --- /dev/null +++ b/tests/rust/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/tests/rust/Cargo.lock b/tests/rust/Cargo.lock new file mode 100644 index 000000000..f24df81e0 --- /dev/null +++ b/tests/rust/Cargo.lock @@ -0,0 +1,1322 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", +] + +[[package]] +name = "allocator-api2" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" + +[[package]] +name = "atoi" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" + +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-intrusive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a604f7a68fbf8103337523b1fadc8ade7361ee3f112f7c680ad179651616aed5" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.11.2", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash 0.8.3", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" +dependencies = [ + "hashbrown 0.14.0", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnetwork" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f84f1612606f3753f205a4e9a2efd6fe5b4c573a6269b2cc6c3003d44a0d127" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.146" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" + +[[package]] +name = "lock_api" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" + +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.8", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.3.5", + "smallvec", + "windows-targets", +] + +[[package]] +name = "paste" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall 0.2.16", + "thiserror", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rust" +version = "0.1.0" +dependencies = [ + "sqlx", + "tokio", +] + +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64 0.21.2", +] + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "serde" +version = "1.0.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "sqlformat" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8de3b03a925878ed54a954f621e64bf55a3c1bd29652d0d1a17830405350188" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + +[[package]] +name = "sqlx-core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" +dependencies = [ + "ahash 0.7.6", + "atoi", + "base64 0.13.1", + "bitflags", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "dirs", + "dotenvy", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-util", + "hashlink", + "hex", + "hkdf", + "hmac", + "indexmap", + "ipnetwork", + "itoa", + "libc", + "log", + "md-5", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rand", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "time", + "tokio-stream", + "url", + "uuid", + "webpki-roots", + "whoami", +] + +[[package]] +name = "sqlx-macros" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966e64ae989e7e575b19d7265cb79d7fc3cbbdf179835cb0d716f294c2049c9" +dependencies = [ + "dotenvy", + "either", + "heck", + "once_cell", + "proc-macro2", + "quote", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-rt", + "syn 1.0.109", + "url", +] + +[[package]] +name = "sqlx-rt" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" +dependencies = [ + "once_cell", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + +[[package]] +name = "time" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +dependencies = [ + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot 0.12.1", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "uuid" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.18", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + +[[package]] +name = "whoami" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c70234412ca409cc04e864e89523cb0fc37f5e1344ebed5a3ebf4192b6b9f68" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/tests/rust/Cargo.toml b/tests/rust/Cargo.toml new file mode 100644 index 000000000..1600dacf3 --- /dev/null +++ b/tests/rust/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rust" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +sqlx = { version = "0.6.2", features = [ "runtime-tokio-rustls", "postgres", "json", "tls", "migrate", "time", "uuid", "ipnetwork"] } +tokio = { version = "1", features = ["full"] } diff --git a/tests/rust/src/main.rs b/tests/rust/src/main.rs new file mode 100644 index 000000000..79667bcc8 --- /dev/null +++ b/tests/rust/src/main.rs @@ -0,0 +1,29 @@ +#[tokio::main] +async fn main() { + test_prepared_statements().await; +} + +async fn test_prepared_statements() { + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .connect("postgres://sharding_user:sharding_user@127.0.0.1:6432/sharded_db") + .await + .unwrap(); + + let mut handles = Vec::new(); + + for _ in 0..5 { + let pool = pool.clone(); + let handle = tokio::task::spawn(async move { + for _ in 0..1000 { + sqlx::query("SELECT 1").fetch_all(&pool).await.unwrap(); + } + }); + + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } +} From 4ce6fb8828f4fb528f12de4f6a9104921812eca7 Mon Sep 17 00:00:00 2001 From: Juraj Bubniak Date: Sat, 17 Jun 2023 03:48:38 +0200 Subject: [PATCH 421/551] Log not implemented messages as debug in prometheus metrics. (#477) --- src/prometheus.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/prometheus.rs b/src/prometheus.rs index b4884604a..7774b5a58 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -1,6 +1,6 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Request, Response, Server, StatusCode}; -use log::{error, info, warn}; +use log::{error, info, debug}; use phf::phf_map; use std::collections::HashMap; use std::fmt; @@ -275,7 +275,7 @@ fn push_address_stats(lines: &mut Vec) { { lines.push(prometheus_metric.to_string()); } else { - warn!("Metric {} not implemented for {}", key, address.name()); + debug!("Metric {} not implemented for {}", key, address.name()); } } } @@ -293,7 +293,7 @@ fn push_pool_stats(lines: &mut Vec) { { lines.push(prometheus_metric.to_string()); } else { - warn!("Metric {} not implemented for ({})", name, *pool_id); + debug!("Metric {} not implemented for ({})", name, *pool_id); } } } @@ -318,7 +318,7 @@ fn push_database_stats(lines: &mut Vec) { { lines.push(prometheus_metric.to_string()); } else { - warn!("Metric {} not implemented for {}", key, address.name()); + debug!("Metric {} not implemented for {}", key, address.name()); } } } From c569a667d6fce9552bc802d5426367df268366b6 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 17 Jun 2023 08:57:33 -0700 Subject: [PATCH 422/551] fmt (#480) --- src/prometheus.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prometheus.rs b/src/prometheus.rs index 7774b5a58..1f070a324 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -1,6 +1,6 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Request, Response, Server, StatusCode}; -use log::{error, info, debug}; +use log::{debug, error, info}; use phf::phf_map; use std::collections::HashMap; use std::fmt; From 07e1376714da2152cdf67d57ce828a0752d4709e Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sat, 17 Jun 2023 09:09:54 -0700 Subject: [PATCH 423/551] Fix build (#481) --- src/prometheus.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prometheus.rs b/src/prometheus.rs index 1f070a324..7e264dcad 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -364,7 +364,7 @@ fn push_server_stats(lines: &mut Vec) { { lines.push(prometheus_metric.to_string()); } else { - warn!("Metric {} not implemented for {}", key, address.name()); + debug!("Metric {} not implemented for {}", key, address.name()); } } } From e5a7ca4124b31f6f20b29c619712dae320bcec5b Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 18 Jun 2023 23:02:34 -0700 Subject: [PATCH 424/551] Implement Close for prepared statements (#482) * Partial support for Close * Close * respect config value * prepared spec * Hmm * Print cache size --- pgcat.toml | 3 ++ src/admin.rs | 5 +++ src/client.rs | 50 ++++++++++++++++++++++- src/config.rs | 24 +++++++++-- src/messages.rs | 80 ++++++++++++++++++++++++++++++++++++- src/server.rs | 76 ++++++++++++++++++++++++++++++++++- src/stats/server.rs | 10 +++++ tests/ruby/prepared_spec.rb | 29 ++++++++++++++ 8 files changed, 269 insertions(+), 8 deletions(-) create mode 100644 tests/ruby/prepared_spec.rb diff --git a/pgcat.toml b/pgcat.toml index 41d0210a0..3e8801b69 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -63,6 +63,9 @@ tcp_keepalives_interval = 5 # Handle prepared statements. prepared_statements = true +# Prepared statements server cache size. +prepared_statements_cache_size = 500 + # Path to TLS Certificate file to use for TLS connections # tls_certificate = ".circleci/server.cert" # Path to TLS private key file to use for TLS connections diff --git a/src/admin.rs b/src/admin.rs index bbca956f9..03b984aee 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -701,6 +701,7 @@ where ("age_seconds", DataType::Numeric), ("prepare_cache_hit", DataType::Numeric), ("prepare_cache_miss", DataType::Numeric), + ("prepare_cache_size", DataType::Numeric), ]; let new_map = get_server_stats(); @@ -732,6 +733,10 @@ where .prepared_miss_count .load(Ordering::Relaxed) .to_string(), + server + .prepared_cache_size + .load(Ordering::Relaxed) + .to_string(), ]; res.put(data_row(&row)); diff --git a/src/client.rs b/src/client.rs index 608d838df..6c0d06fc5 100644 --- a/src/client.rs +++ b/src/client.rs @@ -906,6 +906,19 @@ where return Ok(()); } + // Close (F) + 'C' => { + if prepared_statements_enabled { + let close: Close = (&message).try_into()?; + + if close.is_prepared_statement() && !close.anonymous() { + self.prepared_statements.remove(&close.name); + write_all_flush(&mut self.write, &close_complete()).await?; + continue; + } + } + } + _ => (), } @@ -1130,7 +1143,17 @@ where } else { // The statement is not prepared on the server, so we need to prepare it. if server.should_prepare(&statement.name) { - server.prepare(statement).await?; + match server.prepare(statement).await { + Ok(_) => (), + Err(err) => { + pool.ban( + &address, + BanReason::MessageSendFailed, + Some(&self.stats), + ); + return Err(err); + } + } } } @@ -1251,6 +1274,10 @@ where self.stats.disconnect(); self.release(); + if prepared_statements_enabled { + server.maintain_cache().await?; + } + return Ok(()); } @@ -1300,6 +1327,21 @@ where // Close the prepared statement. 'C' => { + if prepared_statements_enabled { + let close: Close = (&message).try_into()?; + + if close.is_prepared_statement() && !close.anonymous() { + match self.prepared_statements.get(&close.name) { + Some(parse) => { + server.will_close(&parse.generated_name); + } + + // A prepared statement slipped through? Not impossible, since we don't support PREPARE yet. + None => (), + }; + } + } + self.buffer.put(&message[..]); } @@ -1433,7 +1475,13 @@ where // The server is no longer bound to us, we can't cancel it's queries anymore. debug!("Releasing server back into the pool"); + server.checkin_cleanup().await?; + + if prepared_statements_enabled { + server.maintain_cache().await?; + } + server.stats().idle(); self.connected_to_server = false; diff --git a/src/config.rs b/src/config.rs index 66c20758e..a2314fc1b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -323,6 +323,9 @@ pub struct General { #[serde(default)] pub prepared_statements: bool, + + #[serde(default = "General::default_prepared_statements_cache_size")] + pub prepared_statements_cache_size: usize, } impl General { @@ -400,6 +403,10 @@ impl General { pub fn default_server_round_robin() -> bool { true } + + pub fn default_prepared_statements_cache_size() -> usize { + 500 + } } impl Default for General { @@ -438,6 +445,7 @@ impl Default for General { server_round_robin: false, validate_config: true, prepared_statements: false, + prepared_statements_cache_size: 500, } } } @@ -1020,6 +1028,12 @@ impl Config { self.general.verify_server_certificate ); info!("Prepared statements: {}", self.general.prepared_statements); + if self.general.prepared_statements { + info!( + "Prepared statements server cache size: {}", + self.general.prepared_statements_cache_size + ); + } info!( "Plugins: {}", match self.plugins { @@ -1239,13 +1253,15 @@ pub fn get_config() -> Config { } pub fn get_idle_client_in_transaction_timeout() -> u64 { - (*(*CONFIG.load())) - .general - .idle_client_in_transaction_timeout + CONFIG.load().general.idle_client_in_transaction_timeout } pub fn get_prepared_statements() -> bool { - (*(*CONFIG.load())).general.prepared_statements + CONFIG.load().general.prepared_statements +} + +pub fn get_prepared_statements_cache_size() -> usize { + CONFIG.load().general.prepared_statements_cache_size } /// Parse the configuration file located at the path. diff --git a/src/messages.rs b/src/messages.rs index 552497f15..196abe83c 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -1,7 +1,7 @@ /// Helper functions to send one-off protocol messages /// and handle TcpStream (TCP socket). use bytes::{Buf, BufMut, BytesMut}; -use log::error; +use log::{debug, error}; use md5::{Digest, Md5}; use socket2::{SockRef, TcpKeepalive}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -976,6 +976,84 @@ impl Describe { } } +/// Close (F) message. +/// See: +#[derive(Clone, Debug)] +pub struct Close { + code: char, + #[allow(dead_code)] + len: i32, + close_type: char, + pub name: String, +} + +impl TryFrom<&BytesMut> for Close { + type Error = Error; + + fn try_from(bytes: &BytesMut) -> Result { + let mut cursor = Cursor::new(bytes); + let code = cursor.get_u8() as char; + let len = cursor.get_i32(); + let close_type = cursor.get_u8() as char; + let name = cursor.read_string()?; + + Ok(Close { + code, + len, + close_type, + name, + }) + } +} + +impl TryFrom for BytesMut { + type Error = Error; + + fn try_from(close: Close) -> Result { + debug!("Close: {:?}", close); + + let mut bytes = BytesMut::new(); + let name_binding = CString::new(close.name)?; + let name = name_binding.as_bytes_with_nul(); + let len = 4 + 1 + name.len(); + + bytes.put_u8(close.code as u8); + bytes.put_i32(len as i32); + bytes.put_u8(close.close_type as u8); + bytes.put_slice(name); + + Ok(bytes) + } +} + +impl Close { + pub fn new(name: &str) -> Close { + let name = name.to_string(); + + Close { + code: 'C', + len: 4 + 1 + name.len() as i32 + 1, // will be recalculated + close_type: 'S', + name, + } + } + + pub fn is_prepared_statement(&self) -> bool { + self.close_type == 'S' + } + + pub fn anonymous(&self) -> bool { + self.name.is_empty() + } +} + +pub fn close_complete() -> BytesMut { + let mut bytes = BytesMut::new(); + bytes.put_u8(b'3'); + bytes.put_i32(4); + bytes +} + pub fn prepared_statement_name() -> String { format!( "P_{}", diff --git a/src/server.rs b/src/server.rs index ab29db09a..fa68b678b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -15,7 +15,7 @@ use tokio::net::TcpStream; use tokio_rustls::rustls::{OwnedTrustAnchor, RootCertStore}; use tokio_rustls::{client::TlsStream, TlsConnector}; -use crate::config::{get_config, Address, User}; +use crate::config::{get_config, get_prepared_statements_cache_size, Address, User}; use crate::constants::*; use crate::dns_cache::{AddrSet, CACHED_RESOLVER}; use crate::errors::{Error, ServerIdentifier}; @@ -914,12 +914,16 @@ impl Server { Ok(bytes) } + /// Add the prepared statement to being tracked by this server. + /// The client is processing data that will create a prepared statement on this server. pub fn will_prepare(&mut self, name: &str) { debug!("Will prepare `{}`", name); self.prepared_statements.insert(name.to_string()); + self.stats.prepared_cache_add(); } + /// Check if we should prepare a statement on the server. pub fn should_prepare(&self, name: &str) -> bool { let should_prepare = !self.prepared_statements.contains(name); @@ -934,6 +938,7 @@ impl Server { should_prepare } + /// Create a prepared statement on the server. pub async fn prepare(&mut self, parse: &Parse) -> Result<(), Error> { debug!("Preparing `{}`", parse.name); @@ -942,15 +947,82 @@ impl Server { self.send(&flush()).await?; // Read and discard ParseComplete (B) - let _ = read_message(&mut self.stream).await?; + match read_message(&mut self.stream).await { + Ok(_) => (), + Err(err) => { + self.bad = true; + return Err(err); + } + } self.prepared_statements.insert(parse.name.to_string()); + self.stats.prepared_cache_add(); debug!("Prepared `{}`", parse.name); Ok(()) } + /// Maintain adequate cache size on the server. + pub async fn maintain_cache(&mut self) -> Result<(), Error> { + debug!("Cache maintenance run"); + + let max_cache_size = get_prepared_statements_cache_size(); + let mut names = Vec::new(); + + while self.prepared_statements.len() >= max_cache_size { + // The prepared statmeents are alphanumerically sorted by the BTree. + // FIFO. + if let Some(name) = self.prepared_statements.pop_last() { + names.push(name); + } + } + + self.deallocate(names).await?; + + Ok(()) + } + + /// Remove the prepared statement from being tracked by this server. + /// The client is processing data that will cause the server to close the prepared statement. + pub fn will_close(&mut self, name: &str) { + debug!("Will close `{}`", name); + + self.prepared_statements.remove(name); + } + + /// Close a prepared statement on the server. + pub async fn deallocate(&mut self, names: Vec) -> Result<(), Error> { + for name in &names { + debug!("Deallocating prepared statement `{}`", name); + + let close = Close::new(name); + let bytes: BytesMut = close.try_into()?; + + self.send(&bytes).await?; + } + + self.send(&flush()).await?; + + // Read and discard CloseComplete (3) + for name in &names { + match read_message(&mut self.stream).await { + Ok(_) => { + self.prepared_statements.remove(name); + self.stats.prepared_cache_remove(); + debug!("Closed `{}`", name); + } + + Err(err) => { + self.bad = true; + return Err(err); + } + }; + } + + Ok(()) + } + /// If the server is still inside a transaction. /// If the client disconnects while the server is in a transaction, we will clean it up. pub fn in_transaction(&self) -> bool { diff --git a/src/stats/server.rs b/src/stats/server.rs index 6fb2dc973..443c0b6aa 100644 --- a/src/stats/server.rs +++ b/src/stats/server.rs @@ -49,6 +49,7 @@ pub struct ServerStats { pub error_count: Arc, pub prepared_hit_count: Arc, pub prepared_miss_count: Arc, + pub prepared_cache_size: Arc, } impl Default for ServerStats { @@ -67,6 +68,7 @@ impl Default for ServerStats { reporter: get_reporter(), prepared_hit_count: Arc::new(AtomicU64::new(0)), prepared_miss_count: Arc::new(AtomicU64::new(0)), + prepared_cache_size: Arc::new(AtomicU64::new(0)), } } } @@ -213,4 +215,12 @@ impl ServerStats { pub fn prepared_cache_miss(&self) { self.prepared_miss_count.fetch_add(1, Ordering::Relaxed); } + + pub fn prepared_cache_add(&self) { + self.prepared_cache_size.fetch_add(1, Ordering::Relaxed); + } + + pub fn prepared_cache_remove(&self) { + self.prepared_cache_size.fetch_sub(1, Ordering::Relaxed); + } } diff --git a/tests/ruby/prepared_spec.rb b/tests/ruby/prepared_spec.rb new file mode 100644 index 000000000..58a30006b --- /dev/null +++ b/tests/ruby/prepared_spec.rb @@ -0,0 +1,29 @@ +require_relative 'spec_helper' + +describe 'Prepared statements' do + let(:processes) { Helpers::Pgcat.three_shard_setup('sharded_db', 5) } + + context 'enabled' do + it 'will work over the same connection' do + conn = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + + 10.times do |i| + statement_name = "statement_#{i}" + conn.prepare(statement_name, 'SELECT $1::int') + conn.exec_prepared(statement_name, [1]) + conn.describe_prepared(statement_name) + end + end + + it 'will work with new connections' do + 10.times do + conn = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + + statement_name = 'statement1' + conn.prepare('statement1', 'SELECT $1::int') + conn.exec_prepared('statement1', [1]) + conn.describe_prepared('statement1') + end + end + end +end From 16caedfa281cc2e26758fc962a835737cea7a914 Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Fri, 23 Jun 2023 12:15:44 -0400 Subject: [PATCH 425/551] Fixes the default for round robing in General (#488) --- src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index a2314fc1b..ea5a8bfe9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -442,7 +442,7 @@ impl Default for General { auth_query_user: None, auth_query_password: None, server_lifetime: Self::default_server_lifetime(), - server_round_robin: false, + server_round_robin: Self::default_server_round_robin(), validate_config: true, prepared_statements: false, prepared_statements_cache_size: 500, From 6b1cf2d03cd62e15be8bd583b2cbf1c9b006a716 Mon Sep 17 00:00:00 2001 From: Voldemarich Date: Mon, 10 Jul 2023 11:35:43 +0300 Subject: [PATCH 426/551] [BUG] Fix binding of NULL value parameters in prepared statements (#496) Fix binding of NULL value parameters in prepared statements --- src/messages.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/messages.rs b/src/messages.rs index 196abe83c..1f4946588 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -832,10 +832,21 @@ impl TryFrom<&BytesMut> for Bind { for _ in 0..num_param_values { let param_len = cursor.get_i32(); - let mut param = BytesMut::with_capacity(param_len as usize); - param.resize(param_len as usize, b'0'); - cursor.copy_to_slice(&mut param); - param_values.push((param_len, param)); + // There is special occasion when the parameter is NULL + // In that case, param length is defined as -1 + // So if the passed parameter len is over 0 + if param_len > 0 { + let mut param = BytesMut::with_capacity(param_len as usize); + param.resize(param_len as usize, b'0'); + cursor.copy_to_slice(&mut param); + // we push and the length and the parameter into vector + param_values.push((param_len, param)); + } else { + // otherwise we push a tuple with -1 and 0-len BytesMut + // which means that after encountering -1 postgres proceeds + // to processing another parameter + param_values.push((param_len, BytesMut::new())); + } } let num_result_column_format_codes = cursor.get_i16(); From 5cb416221831f17ff49bb104dc8356ca0e707e97 Mon Sep 17 00:00:00 2001 From: Cluas Date: Wed, 12 Jul 2023 20:42:44 +0800 Subject: [PATCH 427/551] docs: fix general.autoreload description (#491) * docs: fix autoreload description Signed-off-by: Cluas * docs: add blank line Signed-off-by: Cluas --------- Signed-off-by: Cluas --- CONFIG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index b36a190df..7a1a86742 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -1,4 +1,4 @@ -# PgCat Configurations +# PgCat Configurations ## `general` Section ### host @@ -116,10 +116,10 @@ If we should log client disconnections ### autoreload ``` path: general.autoreload -default: 15000 +default: 15000 # milliseconds ``` -When set to true, PgCat reloads configs if it detects a change in the config file. +When set, PgCat automatically reloads its configurations at the specified interval (in milliseconds) if it detects changes in the configuration file. The default interval is 15000 milliseconds or 15 seconds. ### worker_threads ``` From c2dcef4f29929ef4357b6514fd9f8b10e99df76a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 05:46:31 -0700 Subject: [PATCH 428/551] chore(deps): bump rustls from 0.21.1 to 0.21.5 (#501) Bumps [rustls](https://github.com/rustls/rustls) from 0.21.1 to 0.21.5. - [Release notes](https://github.com/rustls/rustls/releases) - [Commits](https://github.com/rustls/rustls/compare/v/0.21.1...v/0.21.5) --- updated-dependencies: - dependency-name: rustls dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0baef1ced..b5a43b8f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1159,13 +1159,13 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.1" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" dependencies = [ "log", "ring", - "rustls-webpki", + "rustls-webpki 0.101.1", "sct", ] @@ -1188,6 +1188,16 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.101.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.13" @@ -1794,7 +1804,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa54963694b65584e170cf5dc46aeb4dcaa5584e652ff5f3952e56d66aff0125" dependencies = [ - "rustls-webpki", + "rustls-webpki 0.100.1", ] [[package]] From 515c7edb97bf1b6c6e2503ab9078fbe815ec1fa9 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 12 Jul 2023 13:24:30 -0500 Subject: [PATCH 429/551] Add support for tcp_user_timeout (#503) * Add support for tcp_user_timeout * option * duration * Some() * docs * fmt, compile --- CONFIG.md | 8 +++++++- src/config.rs | 7 +++++++ src/messages.rs | 9 ++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 7a1a86742..3118a49a4 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -151,7 +151,13 @@ path: general.tcp_keepalives_interval default: 5 ``` -Number of seconds between keepalive packets. +### tcp_user_timeout +``` +path: general.tcp_user_timeout +default: 10000 +``` +A linux-only parameters that defines the amount of time in milliseconds that transmitted data may remain unacknowledged or buffered data may remain untransmitted (due to zero window size) before TCP will forcibly disconnect + ### tls_certificate ``` diff --git a/src/config.rs b/src/config.rs index ea5a8bfe9..9228b9bbe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -261,6 +261,8 @@ pub struct General { pub tcp_keepalives_count: u32, #[serde(default = "General::default_tcp_keepalives_interval")] pub tcp_keepalives_interval: u64, + #[serde(default = "General::default_tcp_user_timeout")] + pub tcp_user_timeout: u64, #[serde(default)] // False pub log_client_connections: bool, @@ -360,6 +362,10 @@ impl General { 5 // 5 seconds } + pub fn default_tcp_user_timeout() -> u64 { + 10000 // 10000 milliseconds + } + pub fn default_idle_timeout() -> u64 { 600000 // 10 minutes } @@ -427,6 +433,7 @@ impl Default for General { tcp_keepalives_idle: Self::default_tcp_keepalives_idle(), tcp_keepalives_count: Self::default_tcp_keepalives_count(), tcp_keepalives_interval: Self::default_tcp_keepalives_interval(), + tcp_user_timeout: Self::default_tcp_user_timeout(), log_client_connections: false, log_client_disconnections: false, autoreload: None, diff --git a/src/messages.rs b/src/messages.rs index 1f4946588..eb0ea73c7 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -669,6 +669,13 @@ pub fn configure_socket(stream: &TcpStream) { let sock_ref = SockRef::from(stream); let conf = get_config(); + #[cfg(target_os = "linux")] + match sock_ref.set_tcp_user_timeout(Some(Duration::from_millis(conf.general.tcp_user_timeout))) + { + Ok(_) => (), + Err(err) => error!("Could not configure tcp_user_timeout for socket: {}", err), + } + match sock_ref.set_keepalive(true) { Ok(_) => { match sock_ref.set_tcp_keepalive( @@ -678,7 +685,7 @@ pub fn configure_socket(stream: &TcpStream) { .with_time(Duration::from_secs(conf.general.tcp_keepalives_idle)), ) { Ok(_) => (), - Err(err) => error!("Could not configure socket: {}", err), + Err(err) => error!("Could not configure tcp_keepalive for socket: {}", err), } } Err(err) => error!("Could not configure socket: {}", err), From df0fd5867d81bf80d0646e46bdcdac6a39e4fed9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 21:41:48 -0700 Subject: [PATCH 430/551] chore(deps): bump rustls-pemfile from 1.0.2 to 1.0.3 (#504) Bumps [rustls-pemfile](https://github.com/rustls/pemfile) from 1.0.2 to 1.0.3. - [Commits](https://github.com/rustls/pemfile/commits) --- updated-dependencies: - dependency-name: rustls-pemfile dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b5a43b8f0..05597050f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1171,9 +1171,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ "base64", ] From 92140f69131c590b9f15aab24cc1d00fd023850e Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Fri, 14 Jul 2023 02:40:04 -0300 Subject: [PATCH 431/551] add "show help" command (#505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds a new function to handle notify and use it in the SHOW HELP command, which displays the available options in the admin console. Also, adding Fabrízio as a co-author for all the help with the protocol and the help to structure this PR. Signed-off-by: Sebastian Webber --- src/admin.rs | 45 ++++++++++++++++++++++++++++++++++++++++++++- src/messages.rs | 20 ++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/admin.rs b/src/admin.rs index 03b984aee..d17659e87 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,7 +1,7 @@ use crate::pool::BanReason; use crate::stats::pool::PoolStats; use bytes::{Buf, BufMut, BytesMut}; -use log::{error, info, trace}; +use log::{debug, error, info, trace}; use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; use std::collections::HashMap; @@ -84,6 +84,10 @@ where shutdown(stream).await } "SHOW" => match query_parts[1].to_ascii_uppercase().as_str() { + "HELP" => { + trace!("SHOW HELP"); + show_help(stream).await + } "BANS" => { trace!("SHOW BANS"); show_bans(stream).await @@ -271,6 +275,45 @@ where write_all_half(stream, &res).await } +/// Show all available options. +async fn show_help(stream: &mut T) -> Result<(), Error> +where + T: tokio::io::AsyncWrite + std::marker::Unpin, +{ + let mut res = BytesMut::new(); + + let detail_msg = vec![ + "", + "SHOW HELP|CONFIG|DATABASES|POOLS|CLIENTS|SERVERS|USERS|VERSION", + // "SHOW PEERS|PEER_POOLS", // missing PEERS|PEER_POOLS + // "SHOW FDS|SOCKETS|ACTIVE_SOCKETS|LISTS|MEM|STATE", // missing FDS|SOCKETS|ACTIVE_SOCKETS|MEM|STATE + "SHOW LISTS", + // "SHOW DNS_HOSTS|DNS_ZONES", // missing DNS_HOSTS|DNS_ZONES + "SHOW STATS", // missing STATS_TOTALS|STATS_AVERAGES|TOTALS + "SET key = arg", + "RELOAD", + "PAUSE [, ]", + "RESUME [, ]", + // "DISABLE ", // missing + // "ENABLE ", // missing + // "RECONNECT []", missing + // "KILL ", + // "SUSPEND", + "SHUTDOWN", + // "WAIT_CLOSE []", // missing + ]; + + res.put(notify("Console usage", detail_msg.join("\n\t"))); + res.put(command_complete("SHOW")); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await +} + /// Show shards and replicas. async fn show_databases(stream: &mut T) -> Result<(), Error> where diff --git a/src/messages.rs b/src/messages.rs index eb0ea73c7..8ebc00a34 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -530,6 +530,26 @@ pub fn command_complete(command: &str) -> BytesMut { res } +/// Create a notify message. +pub fn notify(message: &str, details: String) -> BytesMut { + let mut notify_cmd = BytesMut::new(); + + notify_cmd.put_slice("SNOTICE\0".as_bytes()); + notify_cmd.put_slice("C00000\0".as_bytes()); + notify_cmd.put_slice(format!("M{}\0", message).as_bytes()); + notify_cmd.put_slice(format!("D{}\0", details).as_bytes()); + + // this extra byte says that is the end of the package + notify_cmd.put_u8(0); + + let mut res = BytesMut::new(); + res.put_u8(b'N'); + res.put_i32(notify_cmd.len() as i32 + 4); + res.put(notify_cmd); + + res +} + pub fn flush() -> BytesMut { let mut bytes = BytesMut::new(); bytes.put_u8(b'H'); From 0c299345df902a392dee7da28961c187f89b3cfc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jul 2023 07:58:11 -0700 Subject: [PATCH 432/551] chore(deps): bump num_cpus from 1.15.0 to 1.16.0 (#507) Bumps [num_cpus](https://github.com/seanmonstar/num_cpus) from 1.15.0 to 1.16.0. - [Release notes](https://github.com/seanmonstar/num_cpus/releases) - [Changelog](https://github.com/seanmonstar/num_cpus/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/num_cpus/compare/v1.15.0...v1.16.0) --- updated-dependencies: - dependency-name: num_cpus dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05597050f..80d0ba558 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -470,15 +470,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.1" @@ -658,7 +649,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi", "io-lifetimes", "rustix", "windows-sys", @@ -849,11 +840,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] From ed436f83933fdfc4147461fd374a9e610f30e72b Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Fri, 14 Jul 2023 14:30:49 -0300 Subject: [PATCH 433/551] push and build only in main and tags (#508) this commit changes the CI behavior to only build and push when something is committed to main or is a new tag. --- .github/workflows/build-and-push.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index 1baa93bea..953bee9ef 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -1,6 +1,11 @@ name: Build and Push -on: push +on: + push: + branches: + - main + tags: + - v* env: registry: ghcr.io From 30e53af57d1c4774cd7f02fc083e0f483a5addb0 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Tue, 18 Jul 2023 17:52:40 -0300 Subject: [PATCH 434/551] Add cmd line parser (#512) This commit adds the clap library and configures the necessary args to parse from the command line, expanding the current option of a single file and adding support for environment variables. Signed-off-by: Sebastian Webber --- Cargo.lock | 756 ++++++++++++++++++++++++++------------------ Cargo.toml | 1 + src/admin.rs | 2 +- src/cmd_args.rs | 17 + src/main.rs | 15 +- src/multi_logger.rs | 15 +- 6 files changed, 479 insertions(+), 327 deletions(-) create mode 100644 src/cmd_args.rs diff --git a/Cargo.lock b/Cargo.lock index 80d0ba558..b0d455ed7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,36 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -20,6 +41,55 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "arc-swap" version = "1.6.0" @@ -28,34 +98,35 @@ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "async-stream" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", + "pin-project-lite", ] [[package]] name = "async-stream-impl" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.26", ] [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.9", + "syn 2.0.26", ] [[package]] @@ -75,11 +146,26 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" -version = "0.21.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bb8" @@ -100,20 +186,26 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "block-buffer" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -141,13 +233,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "time", "wasm-bindgen", @@ -155,95 +247,88 @@ dependencies = [ ] [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "clap" +version = "4.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +checksum = "8f644d0dac522c8b05ddc39aaaccc5b136d5dc4ff216610c5641e3be5becf56c" dependencies = [ - "termcolor", - "unicode-width", + "clap_builder", + "clap_derive", + "once_cell", ] [[package]] -name = "core-foundation-sys" -version = "0.8.3" +name = "clap_builder" +version = "4.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "af410122b9778e024f9e0fb35682cc09cc3f85cad5e8d3ba8f47a9702df6e73d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] [[package]] -name = "cpufeatures" -version = "0.2.5" +name = "clap_derive" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ - "libc", + "heck", + "proc-macro2", + "quote", + "syn 2.0.26", ] [[package]] -name = "crypto-common" -version = "0.1.6" +name = "clap_lex" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] -name = "cxx" -version = "1.0.91" +name = "colorchoice" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] -name = "cxx-build" -version = "1.0.91" +name = "core-foundation-sys" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 1.0.109", -] +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] -name = "cxxbridge-flags" -version = "1.0.91" +name = "cpufeatures" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +dependencies = [ + "libc", +] [[package]] -name = "cxxbridge-macro" -version = "1.0.91" +name = "crypto-common" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "generic-array", + "typenum", ] [[package]] name = "data-encoding" -version = "2.3.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -281,15 +366,21 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" -version = "0.2.8" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys", ] [[package]] @@ -322,9 +413,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -385,7 +476,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.9", + "syn 2.0.26", ] [[package]] @@ -420,9 +511,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -430,20 +521,26 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "h2" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes", "fnv", @@ -451,7 +548,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -464,17 +561,23 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "hmac" @@ -538,9 +641,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -553,7 +656,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -562,26 +665,25 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", ] [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] @@ -597,9 +699,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -607,50 +709,49 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] -name = "io-lifetimes" -version = "1.0.5" +name = "indexmap" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ - "libc", - "windows-sys", + "equivalent", + "hashbrown 0.14.0", ] [[package]] name = "ipconfig" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd302af1b90f2463a98fa5ad469fc212c8e3175a41c3068601bfa2727591c5be" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2", + "socket2 0.5.3", "widestring", - "winapi", + "windows-sys", "winreg", ] [[package]] name = "ipnet" -version = "2.5.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" -version = "0.4.4" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "io-lifetimes", "rustix", "windows-sys", ] @@ -666,9 +767,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jemalloc-sys" @@ -692,9 +793,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -707,18 +808,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" - -[[package]] -name = "link-cplusplus" -version = "1.0.8" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "linked-hash-map" @@ -728,15 +820,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -765,9 +857,9 @@ checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "md-5" @@ -793,14 +885,22 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys", ] @@ -811,7 +911,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "memoffset", @@ -819,16 +919,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.15" @@ -848,11 +938,20 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "parking_lot" @@ -866,22 +965,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-targets", ] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pgcat" @@ -894,6 +993,7 @@ dependencies = [ "bb8", "bytes", "chrono", + "clap", "env_logger", "exitcode", "fallible-iterator", @@ -920,7 +1020,7 @@ dependencies = [ "serde_json", "sha-1", "sha2", - "socket2", + "socket2 0.4.9", "sqlparser", "stringprep", "tokio", @@ -933,9 +1033,9 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros", "phf_shared", @@ -943,9 +1043,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", "rand", @@ -953,51 +1053,51 @@ dependencies = [ [[package]] name = "phf_macros" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92aacdc5f16768709a569e913f7451034034178b05bdc8acda226659a3dccc66" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator", "phf_shared", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.26", ] [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] [[package]] name = "pin-project" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.9", + "syn 2.0.26", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" [[package]] name = "pin-utils" @@ -1031,9 +1131,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.53" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -1046,9 +1146,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.26" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" dependencies = [ "proc-macro2", ] @@ -1085,18 +1185,30 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.8.4" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" dependencies = [ "aho-corasick", "memchr", @@ -1105,9 +1217,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "resolv-conf" @@ -1134,15 +1246,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustix" -version = "0.36.8" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" dependencies = [ - "bitflags", + "bitflags 2.3.3", "errno", - "io-lifetimes", "libc", "linux-raw-sys", "windows-sys", @@ -1191,21 +1308,15 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -1219,29 +1330,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.160" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn 2.0.9", + "syn 2.0.26", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" dependencies = [ "itoa", "ryu", @@ -1250,9 +1361,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" dependencies = [ "serde", ] @@ -1270,9 +1381,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", @@ -1305,20 +1416,30 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "spin" version = "0.5.2" @@ -1354,19 +1475,25 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "stringprep" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" dependencies = [ "unicode-bidi", "unicode-normalization", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -1381,9 +1508,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.9" +version = "2.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0da4a3c17e109f700685ec577c0f85efd9b19bcf15c913985f14dc1ac01775aa" +checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" dependencies = [ "proc-macro2", "quote", @@ -1401,22 +1528,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.26", ] [[package]] @@ -1447,40 +1574,40 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.26.0" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", + "backtrace", "bytes", "libc", - "memchr", "mio", "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.4.9", "tokio-macros", "windows-sys", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.26", ] [[package]] name = "tokio-rustls" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", "tokio", @@ -1488,9 +1615,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -1512,9 +1639,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -1526,9 +1653,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" dependencies = [ "serde", "serde_spanned", @@ -1538,20 +1665,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.9" +version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d964908cec0d030b812013af25a0e57fddfadb1e066ecc6681d86253129d4f" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap", + "indexmap 2.0.0", "serde", "serde_spanned", "toml_datetime", @@ -1578,20 +1705,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.26", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] @@ -1655,15 +1782,15 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" -version = "0.3.10" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -1674,12 +1801,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - [[package]] name = "untrusted" version = "0.7.1" @@ -1688,15 +1809,21 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", - "idna 0.3.0", + "idna 0.4.0", "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "version_check" version = "0.9.4" @@ -1705,11 +1832,10 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -1727,9 +1853,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1737,24 +1863,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.26", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1762,28 +1888,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.26", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -1791,18 +1917,18 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.23.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa54963694b65584e170cf5dc46aeb4dcaa5584e652ff5f3952e56d66aff0125" +checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ "rustls-webpki 0.100.1", ] [[package]] name = "widestring" -version = "0.5.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" +checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" [[package]] name = "winapi" @@ -1835,20 +1961,29 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" -version = "0.45.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.1" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -1861,60 +1996,61 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.6" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +checksum = "81fac9742fd1ad1bd9643b991319f72dd031016d44b77039a26977eb667141e7" dependencies = [ "memchr", ] [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if", + "windows-sys", ] diff --git a/Cargo.toml b/Cargo.toml index a70c7e938..e778d8308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ trust-dns-resolver = "0.22.0" tokio-test = "0.4.2" serde_json = "1" itertools = "0.10" +clap = { version = "4.3.1", features = ["derive", "env"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/src/admin.rs b/src/admin.rs index d17659e87..6c83f9b20 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,7 +1,7 @@ use crate::pool::BanReason; use crate::stats::pool::PoolStats; use bytes::{Buf, BufMut, BytesMut}; -use log::{debug, error, info, trace}; +use log::{error, info, trace}; use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; use std::collections::HashMap; diff --git a/src/cmd_args.rs b/src/cmd_args.rs new file mode 100644 index 000000000..159eefe9c --- /dev/null +++ b/src/cmd_args.rs @@ -0,0 +1,17 @@ +use clap::Parser; +use log::LevelFilter; + +/// PgCat: Nextgen PostgreSQL Pooler +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +pub(crate) struct Args { + #[arg(default_value_t = String::from("pgcat.toml"), env)] + pub config_file: String, + + #[arg(short, long, default_value_t = LevelFilter::Info, env)] + pub log_level: log::LevelFilter, +} + +pub(crate) fn parse() -> Args { + return Args::parse(); +} diff --git a/src/main.rs b/src/main.rs index 6af4db95b..2b3294ee4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -68,8 +68,11 @@ use pgcat::pool::{ClientServerMap, ConnectionPool}; use pgcat::prometheus::start_metric_server; use pgcat::stats::{Collector, Reporter, REPORTER}; +mod cmd_args; + fn main() -> Result<(), Box> { - pgcat::multi_logger::MultiLogger::init().unwrap(); + let args = cmd_args::parse(); + pgcat::multi_logger::MultiLogger::init(args.log_level).unwrap(); info!("Welcome to PgCat! Meow. (Version {})", VERSION); @@ -78,20 +81,12 @@ fn main() -> Result<(), Box> { std::process::exit(exitcode::CONFIG); } - let args = std::env::args().collect::>(); - - let config_file = if args.len() == 2 { - args[1].to_string() - } else { - String::from("pgcat.toml") - }; - // Create a transient runtime for loading the config for the first time. { let runtime = Builder::new_multi_thread().worker_threads(1).build()?; runtime.block_on(async { - match pgcat::config::parse(&config_file).await { + match pgcat::config::parse(args.config_file.as_str()).await { Ok(_) => (), Err(err) => { error!("Config parse error: {:?}", err); diff --git a/src/multi_logger.rs b/src/multi_logger.rs index 2a61794e4..8c620698a 100644 --- a/src/multi_logger.rs +++ b/src/multi_logger.rs @@ -1,4 +1,4 @@ -use log::{Level, Log, Metadata, Record, SetLoggerError}; +use log::{Level, LevelFilter, Log, Metadata, Record, SetLoggerError}; // This is a special kind of logger that allows sending logs to different // targets depending on the log level. @@ -25,8 +25,11 @@ pub struct MultiLogger { } impl MultiLogger { - fn new() -> Self { - let stderr_logger = env_logger::builder().format_timestamp_micros().build(); + fn new(filter: LevelFilter) -> Self { + let stderr_logger = env_logger::builder() + .filter(None, filter) + .format_timestamp_micros() + .build(); let stdout_logger = env_logger::Builder::from_env("STDOUT_LOG") .format_timestamp_micros() .target(env_logger::Target::Stdout) @@ -38,8 +41,8 @@ impl MultiLogger { } } - pub fn init() -> Result<(), SetLoggerError> { - let logger = Self::new(); + pub fn init(filter: LevelFilter) -> Result<(), SetLoggerError> { + let logger = Self::new(filter); log::set_max_level(logger.stderr_logger.filter()); log::set_boxed_logger(Box::new(logger)) @@ -75,6 +78,6 @@ mod test { #[test] fn test_init() { - MultiLogger::init().unwrap(); + MultiLogger::init(LevelFilter::Error).unwrap(); } } From 242f0001b15cad075db5fb73eb502810d7db6781 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 19 Jul 2023 03:07:13 -0300 Subject: [PATCH 435/551] add support for multiple log formats (#517) this commit adds the tracing-subscriber crate and use its formatters to support multiple log formats. More details in https://github.com/postgresml/pgcat/issues/464#issuecomment-1641430299 Signed-off-by: Sebastian Webber --- Cargo.lock | 82 ++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/cmd_args.rs | 22 ++++++++---- src/lib.rs | 2 +- src/main.rs | 19 +++++++++-- src/multi_logger.rs | 83 --------------------------------------------- 6 files changed, 117 insertions(+), 93 deletions(-) delete mode 100644 src/multi_logger.rs diff --git a/Cargo.lock b/Cargo.lock index b0d455ed7..dc9869b6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -919,6 +919,16 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -953,6 +963,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1027,6 +1043,8 @@ dependencies = [ "tokio-rustls", "tokio-test", "toml", + "tracing", + "tracing-subscriber", "trust-dns-resolver", "webpki-roots", ] @@ -1390,6 +1408,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1546,6 +1573,16 @@ dependencies = [ "syn 2.0.26", ] +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "time" version = "0.1.45" @@ -1721,6 +1758,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +dependencies = [ + "nu-ansi-term", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -1824,6 +1900,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index e778d8308..a1c09ab2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,8 @@ tokio-test = "0.4.2" serde_json = "1" itertools = "0.10" clap = { version = "4.3.1", features = ["derive", "env"] } +tracing = "0.1.37" +tracing-subscriber = { version = "0.3.17", features = ["json"]} [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/src/cmd_args.rs b/src/cmd_args.rs index 159eefe9c..08782f471 100644 --- a/src/cmd_args.rs +++ b/src/cmd_args.rs @@ -1,17 +1,27 @@ -use clap::Parser; -use log::LevelFilter; +use clap::{Parser, ValueEnum}; +use tracing::Level; /// PgCat: Nextgen PostgreSQL Pooler #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] -pub(crate) struct Args { +pub struct Args { #[arg(default_value_t = String::from("pgcat.toml"), env)] pub config_file: String, - #[arg(short, long, default_value_t = LevelFilter::Info, env)] - pub log_level: log::LevelFilter, + #[arg(short, long, default_value_t = tracing::Level::INFO, env)] + pub log_level: Level, + + #[clap(short='F', long, value_enum, default_value_t=LogFormat::Text, env)] + pub log_format: LogFormat, } -pub(crate) fn parse() -> Args { +pub fn parse() -> Args { return Args::parse(); } + +#[derive(ValueEnum, Clone, Debug)] +pub enum LogFormat { + Text, + Structured, + Debug, +} diff --git a/src/lib.rs b/src/lib.rs index db6167db0..a0a71ebb4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,13 @@ pub mod admin; pub mod auth_passthrough; pub mod client; +pub mod cmd_args; pub mod config; pub mod constants; pub mod dns_cache; pub mod errors; pub mod messages; pub mod mirrors; -pub mod multi_logger; pub mod plugins; pub mod pool; pub mod prometheus; diff --git a/src/main.rs b/src/main.rs index 2b3294ee4..a6b3aab24 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,18 +61,31 @@ use std::str::FromStr; use std::sync::Arc; use tokio::sync::broadcast; +use pgcat::cmd_args; +use pgcat::cmd_args::LogFormat; use pgcat::config::{get_config, reload_config, VERSION}; use pgcat::dns_cache; use pgcat::messages::configure_socket; use pgcat::pool::{ClientServerMap, ConnectionPool}; use pgcat::prometheus::start_metric_server; use pgcat::stats::{Collector, Reporter, REPORTER}; - -mod cmd_args; +use tracing_subscriber; fn main() -> Result<(), Box> { let args = cmd_args::parse(); - pgcat::multi_logger::MultiLogger::init(args.log_level).unwrap(); + match args.log_format { + LogFormat::Structured => tracing_subscriber::fmt() + .json() + .with_max_level(args.log_level) + .init(), + LogFormat::Debug => tracing_subscriber::fmt() + .pretty() + .with_max_level(args.log_level) + .init(), + _ => tracing_subscriber::fmt() + .with_max_level(args.log_level) + .init(), + }; info!("Welcome to PgCat! Meow. (Version {})", VERSION); diff --git a/src/multi_logger.rs b/src/multi_logger.rs deleted file mode 100644 index 8c620698a..000000000 --- a/src/multi_logger.rs +++ /dev/null @@ -1,83 +0,0 @@ -use log::{Level, LevelFilter, Log, Metadata, Record, SetLoggerError}; - -// This is a special kind of logger that allows sending logs to different -// targets depending on the log level. -// -// By default, if nothing is set, it acts as a regular env_log logger, -// it sends everything to standard error. -// -// If the Env variable `STDOUT_LOG` is defined, it will be used for -// configuring the standard out logger. -// -// The behavior is: -// - If it is an error, the message is written to standard error. -// - If it is not, and it matches the log level of the standard output logger (`STDOUT_LOG` env var), it will be send to standard output. -// - If the above is not true, it is sent to the stderr logger that will log it or not depending on the value -// of the RUST_LOG env var. -// -// So to summarize, if no `STDOUT_LOG` env var is present, the logger is the default logger. If `STDOUT_LOG` is set, everything -// but errors, that matches the log level set in the `STDOUT_LOG` env var is sent to stdout. You can have also some esoteric configuration -// where you set `RUST_LOG=debug` and `STDOUT_LOG=info`, in here, errors will go to stderr, warns and infos to stdout and debugs to stderr. -// -pub struct MultiLogger { - stderr_logger: env_logger::Logger, - stdout_logger: env_logger::Logger, -} - -impl MultiLogger { - fn new(filter: LevelFilter) -> Self { - let stderr_logger = env_logger::builder() - .filter(None, filter) - .format_timestamp_micros() - .build(); - let stdout_logger = env_logger::Builder::from_env("STDOUT_LOG") - .format_timestamp_micros() - .target(env_logger::Target::Stdout) - .build(); - - Self { - stderr_logger, - stdout_logger, - } - } - - pub fn init(filter: LevelFilter) -> Result<(), SetLoggerError> { - let logger = Self::new(filter); - - log::set_max_level(logger.stderr_logger.filter()); - log::set_boxed_logger(Box::new(logger)) - } -} - -impl Log for MultiLogger { - fn enabled(&self, metadata: &Metadata) -> bool { - self.stderr_logger.enabled(metadata) && self.stdout_logger.enabled(metadata) - } - - fn log(&self, record: &Record) { - if record.level() == Level::Error { - self.stderr_logger.log(record); - } else { - if self.stdout_logger.matches(record) { - self.stdout_logger.log(record); - } else { - self.stderr_logger.log(record); - } - } - } - - fn flush(&self) { - self.stderr_logger.flush(); - self.stdout_logger.flush(); - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_init() { - MultiLogger::init(LevelFilter::Error).unwrap(); - } -} From dfcce7201e5d111eaf7bb56e850a34a2ef6da708 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Thu, 20 Jul 2023 01:15:55 -0300 Subject: [PATCH 436/551] add `--no-color` option to disable colors in the terminal (#518) add --no-color option to disable colors this commit adds a new option to disable colors in the terminal and also moves the logger configuration to a different crate. Signed-off-by: Sebastian Webber --- src/cmd_args.rs | 9 +++++++++ src/lib.rs | 1 + src/logger.rs | 14 ++++++++++++++ src/main.rs | 17 ++--------------- 4 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 src/logger.rs diff --git a/src/cmd_args.rs b/src/cmd_args.rs index 08782f471..3989d670f 100644 --- a/src/cmd_args.rs +++ b/src/cmd_args.rs @@ -13,6 +13,15 @@ pub struct Args { #[clap(short='F', long, value_enum, default_value_t=LogFormat::Text, env)] pub log_format: LogFormat, + + #[arg( + short, + long, + default_value_t = false, + env, + help = "disable colors in the log output" + )] + pub no_color: bool, } pub fn parse() -> Args { diff --git a/src/lib.rs b/src/lib.rs index a0a71ebb4..6a8a1e363 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ pub mod config; pub mod constants; pub mod dns_cache; pub mod errors; +pub mod logger; pub mod messages; pub mod mirrors; pub mod plugins; diff --git a/src/logger.rs b/src/logger.rs new file mode 100644 index 000000000..c3536a0cf --- /dev/null +++ b/src/logger.rs @@ -0,0 +1,14 @@ +use crate::cmd_args::{Args, LogFormat}; +use tracing_subscriber; + +pub fn init(args: &Args) { + let trace_sub = tracing_subscriber::fmt() + .with_max_level(args.log_level) + .with_ansi(!args.no_color); + + match args.log_format { + LogFormat::Structured => trace_sub.json().init(), + LogFormat::Debug => trace_sub.pretty().init(), + _ => trace_sub.init(), + }; +} diff --git a/src/main.rs b/src/main.rs index a6b3aab24..916d1a8eb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -62,30 +62,17 @@ use std::sync::Arc; use tokio::sync::broadcast; use pgcat::cmd_args; -use pgcat::cmd_args::LogFormat; use pgcat::config::{get_config, reload_config, VERSION}; use pgcat::dns_cache; +use pgcat::logger; use pgcat::messages::configure_socket; use pgcat::pool::{ClientServerMap, ConnectionPool}; use pgcat::prometheus::start_metric_server; use pgcat::stats::{Collector, Reporter, REPORTER}; -use tracing_subscriber; fn main() -> Result<(), Box> { let args = cmd_args::parse(); - match args.log_format { - LogFormat::Structured => tracing_subscriber::fmt() - .json() - .with_max_level(args.log_level) - .init(), - LogFormat::Debug => tracing_subscriber::fmt() - .pretty() - .with_max_level(args.log_level) - .init(), - _ => tracing_subscriber::fmt() - .with_max_level(args.log_level) - .init(), - }; + logger::init(&args); info!("Welcome to PgCat! Meow. (Version {})", VERSION); From 7443573af878bba823f3c99dfd28a2c52b0f94a7 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 21 Jul 2023 01:06:01 -0500 Subject: [PATCH 437/551] Fix COPY FROM and add tests (#522) * Fix COPY FROM and add tests * E * fmt --- src/client.rs | 4 +- src/server.rs | 29 +++++++++++- tests/ruby/copy_spec.rb | 102 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 tests/ruby/copy_spec.rb diff --git a/src/client.rs b/src/client.rs index 6c0d06fc5..7d5e97987 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1260,7 +1260,7 @@ where // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. - if self.transaction_mode { + if self.transaction_mode && !server.in_copy_mode() { self.stats.idle(); break; @@ -1410,7 +1410,7 @@ where // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. - if self.transaction_mode { + if self.transaction_mode && !server.in_copy_mode() { break; } } diff --git a/src/server.rs b/src/server.rs index fa68b678b..55444fba9 100644 --- a/src/server.rs +++ b/src/server.rs @@ -170,6 +170,9 @@ pub struct Server { /// Is there more data for the client to read. data_available: bool, + /// Is the server in copy-in or copy-out modes + in_copy_mode: bool, + /// Is the server broken? We'll remote it from the pool if so. bad: bool, @@ -677,6 +680,7 @@ impl Server { process_id, secret_key, in_transaction: false, + in_copy_mode: false, data_available: false, bad: false, cleanup_state: CleanupState::new(), @@ -828,8 +832,19 @@ impl Server { break; } + // ErrorResponse + 'E' => { + if self.in_copy_mode { + self.in_copy_mode = false; + } + } + // CommandComplete 'C' => { + if self.in_copy_mode { + self.in_copy_mode = false; + } + let mut command_tag = String::new(); match message.reader().read_to_string(&mut command_tag) { Ok(_) => { @@ -873,10 +888,14 @@ impl Server { } // CopyInResponse: copy is starting from client to server. - 'G' => break, + 'G' => { + self.in_copy_mode = true; + break; + } // CopyOutResponse: copy is starting from the server to the client. 'H' => { + self.in_copy_mode = true; self.data_available = true; break; } @@ -1030,6 +1049,10 @@ impl Server { self.in_transaction } + pub fn in_copy_mode(&self) -> bool { + self.in_copy_mode + } + /// We don't buffer all of server responses, e.g. COPY OUT produces too much data. /// The client is responsible to call `self.recv()` while this method returns true. pub fn is_data_available(&self) -> bool { @@ -1129,6 +1152,10 @@ impl Server { self.cleanup_state.reset(); } + if self.in_copy_mode() { + warn!("Server returned while still in copy-mode"); + } + Ok(()) } diff --git a/tests/ruby/copy_spec.rb b/tests/ruby/copy_spec.rb new file mode 100644 index 000000000..5d3f2c021 --- /dev/null +++ b/tests/ruby/copy_spec.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true +require_relative 'spec_helper' + + +describe "COPY Handling" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 5) } + before do + new_configs = processes.pgcat.current_config + + # Allow connections in the pool to expire faster + new_configs["general"]["idle_timeout"] = 5 + processes.pgcat.update_config(new_configs) + # We need to kill the old process that was using the default configs + processes.pgcat.stop + processes.pgcat.start + processes.pgcat.wait_until_ready + end + + before do + processes.all_databases.first.with_connection do |conn| + conn.async_exec "CREATE TABLE copy_test_table (a TEXT,b TEXT,c TEXT,d TEXT)" + end + end + + after do + processes.all_databases.first.with_connection do |conn| + conn.async_exec "DROP TABLE copy_test_table;" + end + end + + after do + processes.all_databases.map(&:reset) + processes.pgcat.shutdown + end + + describe "COPY FROM" do + context "within transaction" do + it "finishes within alloted time" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + Timeout.timeout(3) do + conn.async_exec("BEGIN") + conn.copy_data "COPY copy_test_table FROM STDIN CSV" do + sleep 0.5 + conn.put_copy_data "some,data,to,copy\n" + conn.put_copy_data "more,data,to,copy\n" + end + conn.async_exec("COMMIT") + end + + res = conn.async_exec("SELECT * FROM copy_test_table").to_a + expect(res).to eq([ + {"a"=>"some", "b"=>"data", "c"=>"to", "d"=>"copy"}, + {"a"=>"more", "b"=>"data", "c"=>"to", "d"=>"copy"} + ]) + end + end + + context "outside transaction" do + it "finishes within alloted time" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + Timeout.timeout(3) do + conn.copy_data "COPY copy_test_table FROM STDIN CSV" do + sleep 0.5 + conn.put_copy_data "some,data,to,copy\n" + conn.put_copy_data "more,data,to,copy\n" + end + end + + res = conn.async_exec("SELECT * FROM copy_test_table").to_a + expect(res).to eq([ + {"a"=>"some", "b"=>"data", "c"=>"to", "d"=>"copy"}, + {"a"=>"more", "b"=>"data", "c"=>"to", "d"=>"copy"} + ]) + end + end + end + + describe "COPY TO" do + before do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.async_exec("BEGIN") + conn.copy_data "COPY copy_test_table FROM STDIN CSV" do + conn.put_copy_data "some,data,to,copy\n" + conn.put_copy_data "more,data,to,copy\n" + end + conn.async_exec("COMMIT") + conn.close + end + + it "works" do + res = [] + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + conn.copy_data "COPY copy_test_table TO STDOUT CSV" do + while row=conn.get_copy_data + res << row + end + end + expect(res).to eq(["some,data,to,copy\n", "more,data,to,copy\n"]) + end + end + +end From be36600992a28f9bd6161f8de055ccc4eb4a5e6e Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 25 Jul 2023 10:27:04 -0700 Subject: [PATCH 438/551] Release 1.1 (#526) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc9869b6a..9bfd4fb63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1000,7 +1000,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pgcat" -version = "1.0.2-alpha3" +version = "1.1.0" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index a1c09ab2a..5f4af3132 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.0.2-alpha3" +version = "1.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From e7e7c20043d3a427b490599948a212c6eb701271 Mon Sep 17 00:00:00 2001 From: Spindel Ljungmark Date: Thu, 27 Jul 2023 17:51:23 +0200 Subject: [PATCH 439/551] Restore the ability to filter spammy log messages (#530) * Move connection checkin log messages to their own target Under heavy load they can happen thousands of times per second, and should generally be considered a nuisance at best. This marks the state discard as an info rather than a warning, and moves all the messages into their own log-target, so they can be filtered separately from the more relevant warnings. Signed-off-by: D.S. Ljungmark * Remove left-over env_logger dependencies When moving to tracing-subscriber for logging, the env_logger dependencies were left around, this cuts them out as dead code. Signed-off-by: D.S. Ljungmark * Restore ability to filter log messages at runtime This restores the RUST_LOG filters from env_logger but now with the tracing subscriber setup. The filters are chained so commandline options mark the default in case either option is set, which should be the path of least confusion for users. ( RUST_LOG setting level to debug, and commandline to warning is an odd user case, and I don't know what a user who does that is expecting. ) It also bumps the version number as a fix to see which versions have which behaviour. Signed-off-by: D.S. Ljungmark --------- Signed-off-by: D.S. Ljungmark --- Cargo.lock | 72 ++++++++++++++++++++++----------------------------- Cargo.toml | 5 ++-- src/logger.rs | 7 ++++- src/main.rs | 1 - src/server.rs | 6 ++--- 5 files changed, 42 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bfd4fb63..d358f0e1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -353,19 +353,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "env_logger" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" -dependencies = [ - "humantime", - "is-terminal", - "log", - "regex", - "termcolor", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -633,12 +620,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - [[package]] name = "hyper" version = "0.14.27" @@ -855,6 +836,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "matches" version = "0.1.10" @@ -1010,7 +1000,6 @@ dependencies = [ "bytes", "chrono", "clap", - "env_logger", "exitcode", "fallible-iterator", "futures", @@ -1218,8 +1207,17 @@ checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.3.3", + "regex-syntax 0.7.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -1230,9 +1228,15 @@ checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.4", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.7.4" @@ -1544,15 +1548,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - [[package]] name = "thiserror" version = "1.0.43" @@ -1788,12 +1783,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "serde", "serde_json", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", "tracing-serde", @@ -2028,15 +2027,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 5f4af3132..c3db48aa7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.1.0" +version = "1.1.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -22,7 +22,6 @@ once_cell = "1" sqlparser = {version = "0.34", features = ["visitor"] } log = "0.4" arc-swap = "1" -env_logger = "0.10" parking_lot = "0.12.1" hmac = "0.12" sha2 = "0.10" @@ -48,7 +47,7 @@ serde_json = "1" itertools = "0.10" clap = { version = "4.3.1", features = ["derive", "env"] } tracing = "0.1.37" -tracing-subscriber = { version = "0.3.17", features = ["json"]} +tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter", "std"]} [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/src/logger.rs b/src/logger.rs index c3536a0cf..f9016ebfa 100644 --- a/src/logger.rs +++ b/src/logger.rs @@ -1,9 +1,14 @@ use crate::cmd_args::{Args, LogFormat}; use tracing_subscriber; +use tracing_subscriber::EnvFilter; pub fn init(args: &Args) { + // Iniitalize a default filter, and then override the builtin default "warning" with our + // commandline, (default: "info") + let filter = EnvFilter::from_default_env().add_directive(args.log_level.into()); + let trace_sub = tracing_subscriber::fmt() - .with_max_level(args.log_level) + .with_env_filter(filter) .with_ansi(!args.no_color); match args.log_format { diff --git a/src/main.rs b/src/main.rs index 916d1a8eb..6c8c16548 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,7 +23,6 @@ extern crate arc_swap; extern crate async_trait; extern crate bb8; extern crate bytes; -extern crate env_logger; extern crate exitcode; extern crate log; extern crate md5; diff --git a/src/server.rs b/src/server.rs index 55444fba9..62cbb3a9c 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1136,7 +1136,7 @@ impl Server { // server connection thrashing if clients repeatedly do this. // Instead, we ROLLBACK that transaction before putting the connection back in the pool if self.in_transaction() { - warn!("Server returned while still in transaction, rolling back transaction"); + warn!(target: "pgcat::server::cleanup", "Server returned while still in transaction, rolling back transaction"); self.query("ROLLBACK").await?; } @@ -1146,14 +1146,14 @@ impl Server { // send `DISCARD ALL` if we think the session is altered instead of just sending // it before each checkin. if self.cleanup_state.needs_cleanup() && self.cleanup_connections { - warn!("Server returned with session state altered, discarding state ({}) for application {}", self.cleanup_state, self.application_name); + info!(target: "pgcat::server::cleanup", "Server returned with session state altered, discarding state ({}) for application {}", self.cleanup_state, self.application_name); self.query("DISCARD ALL").await?; self.query("RESET ROLE").await?; self.cleanup_state.reset(); } if self.in_copy_mode() { - warn!("Server returned while still in copy-mode"); + warn!(target: "pgcat::server::cleanup", "Server returned while still in copy-mode"); } Ok(()) From d585ff4e0dfa423fcc807e40843b0d2463061700 Mon Sep 17 00:00:00 2001 From: Bertrand Paquet Date: Fri, 28 Jul 2023 09:31:53 +0200 Subject: [PATCH 440/551] Fix typo in the config documentation (#532) --- CONFIG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONFIG.md b/CONFIG.md index 3118a49a4..c60e15625 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -230,7 +230,7 @@ default: "random" Load balancing mode `random` selects the server at random -`loc` selects the server with the least outstanding busy conncetions +`loc` selects the server with the least outstanding busy connections ### default_role ``` From b93eb59abc5389379de361d47fd14ec9f7d29566 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 1 Aug 2023 17:46:34 -0700 Subject: [PATCH 441/551] Fix for #534 and #535 --- Cargo.lock | 2 +- pgcat.toml | 2 +- src/server.rs | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d358f0e1f..ec8493293 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -990,7 +990,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pgcat" -version = "1.1.0" +version = "1.1.1" dependencies = [ "arc-swap", "async-trait", diff --git a/pgcat.toml b/pgcat.toml index 3e8801b69..d8ed1088b 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -270,7 +270,7 @@ username = "sharding_user" # if `server_password` is not set. password = "sharding_user" -pool_mode = "session" +pool_mode = "transaction" # PostgreSQL username used to connect to the server. # server_username = "another_user" diff --git a/src/server.rs b/src/server.rs index 62cbb3a9c..bb453aa45 100644 --- a/src/server.rs +++ b/src/server.rs @@ -997,7 +997,9 @@ impl Server { } } - self.deallocate(names).await?; + if !names.is_empty() { + self.deallocate(names).await?; + } Ok(()) } @@ -1013,7 +1015,7 @@ impl Server { /// Close a prepared statement on the server. pub async fn deallocate(&mut self, names: Vec) -> Result<(), Error> { for name in &names { - debug!("Deallocating prepared statement `{}`", name); + info!("Deallocating prepared statement `{}`", name); let close = Close::new(name); let bytes: BytesMut = close.try_into()?; @@ -1021,7 +1023,9 @@ impl Server { self.send(&bytes).await?; } - self.send(&flush()).await?; + if !names.is_empty() { + self.send(&flush()).await?; + } // Read and discard CloseComplete (3) for name in &names { From a4e1fa79bd766a769b81541c9efa26e96f160195 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 1 Aug 2023 17:47:34 -0700 Subject: [PATCH 442/551] debug -> release --- src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index bb453aa45..afa1c09de 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1015,7 +1015,7 @@ impl Server { /// Close a prepared statement on the server. pub async fn deallocate(&mut self, names: Vec) -> Result<(), Error> { for name in &names { - info!("Deallocating prepared statement `{}`", name); + debug!("Deallocating prepared statement `{}`", name); let close = Close::new(name); let bytes: BytesMut = close.try_into()?; From 1cba5f449f02f68671d7c9b73a6ffa67c65b62f6 Mon Sep 17 00:00:00 2001 From: Matthias Pfeil Date: Fri, 4 Aug 2023 18:20:56 +0100 Subject: [PATCH 443/551] Add github tag as tag to image (#537) --- .github/workflows/build-and-push.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index 953bee9ef..545a18f0f 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -34,6 +34,7 @@ jobs: tags: | type=sha,prefix=,format=long type=schedule + type=ref,event=tag type=ref,event=branch type=ref,event=pr type=raw,value=latest,enable={{ is_default_branch }} From 7f8629cd2f0476e03f05b65fe3d59f32234bfa6d Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Aug 2023 11:08:46 -0700 Subject: [PATCH 444/551] pgcat deb package (#539) --- .github/workflows/publish-deb-package.yml | 48 +++++++++++++++++++++++ control | 9 +++++ utilities/deb.sh | 33 ++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 .github/workflows/publish-deb-package.yml create mode 100644 control create mode 100644 utilities/deb.sh diff --git a/.github/workflows/publish-deb-package.yml b/.github/workflows/publish-deb-package.yml new file mode 100644 index 000000000..b46b37daa --- /dev/null +++ b/.github/workflows/publish-deb-package.yml @@ -0,0 +1,48 @@ +name: pgcat package (deb) + +on: + workflow_dispatch: + inputs: + packageVersion: + default: "1.1.1" +jobs: + build: + strategy: + max-parallel: 1 + fail-fast: false # Let the other job finish, or they can lock each other out + matrix: + os: ["buildjet-4vcpu-ubuntu-2204", "buildjet-4vcpu-ubuntu-2204-arm"] + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: Install dependencies + env: + DEBIAN_FRONTEND: noninteractive + TZ: Etc/UTC + run: | + curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem + sudo gem install deb-s3-0.11.4.gem + dpkg-deb --version + - name: Build and release package + env: + AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }} + run: | + if [[ $(arch) == "x86_64" ]]; then + export ARCH=amd64 + else + export ARCH=arm64 + fi + + bash utilities/deb.sh ${{ inputs.packageVersion }} + + deb-s3 upload \ + --lock \ + --bucket apt.postgresml.org \ + pgcat-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb \ + --codename $(lsb_release -cs) diff --git a/control b/control new file mode 100644 index 000000000..f7e5b50ea --- /dev/null +++ b/control @@ -0,0 +1,9 @@ +Package: pgcat +Version: ${PACKAGE_VERSION} +Section: database +Priority: optional +Architecture: ${ARCH} +Maintainer: PostgresML +Homepage: https://postgresml.org +Description: PgCat - NextGen PostgreSQL Pooler + PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load balancing, failover and mirroring. diff --git a/utilities/deb.sh b/utilities/deb.sh new file mode 100644 index 000000000..a194d8038 --- /dev/null +++ b/utilities/deb.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Build an Ubuntu deb. +# +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +deb_dir="/tmp/pgcat-build" +export PACKAGE_VERSION=${1:-"1.1.1"} +if [[ $(arch) == "x86_64" ]]; then + export ARCH=amd64 +else + export ARCH=arm64 +fi + +cd "$script_dir/.." +cargo build --release + +rm -rf "$deb_dir" +mkdir -p "$deb_dir/DEBIAN" +mkdir -p "$deb_dir/usr/bin" +mkdir -p "$deb_dir/etc" + +cp target/release/pgcat "$deb_dir/usr/bin/pgcat" +chmod +x "$deb_dir/usr/bin/pgcat" + +cp pgcat.toml "$deb_dir/etc/pgcat.toml" + +(cat control | envsubst) > "$deb_dir/DEBIAN/control" + +dpkg-deb \ + --root-owner-group \ + -z1 \ + --build "$deb_dir" \ + pgcat-${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb From d601b409201150fccb5ace9d5fdcd3620dbef3a5 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Aug 2023 11:51:38 -0700 Subject: [PATCH 445/551] Add systemd service (#540) --- pgcat.service | 16 ++++++++++++++++ postinst | 4 ++++ postrm | 3 +++ prerm | 4 ++++ utilities/deb.sh | 9 ++++++++- 5 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 pgcat.service create mode 100644 postinst create mode 100644 postrm create mode 100644 prerm diff --git a/pgcat.service b/pgcat.service new file mode 100644 index 000000000..4da4c7c00 --- /dev/null +++ b/pgcat.service @@ -0,0 +1,16 @@ +[Unit] +Description=PgCat pooler +After=network.target +StartLimitIntervalSec=0 + +[Service] +User=pgcat +Type=simple +Restart=always +RestartSec=1 +Environment=RUST_LOG=info +LimitNOFILE=65536 +ExecStart=/usr/bin/pgcat /etc/pgcat.toml + +[Install] +WantedBy=multi-user.target diff --git a/postinst b/postinst new file mode 100644 index 000000000..54c04119b --- /dev/null +++ b/postinst @@ -0,0 +1,4 @@ +#!/bin/bash + +systemctl daemon-reload +systemctl enable pgcat diff --git a/postrm b/postrm new file mode 100644 index 000000000..ab20f7b0f --- /dev/null +++ b/postrm @@ -0,0 +1,3 @@ +#!/bin/bash + +systemctl daemon-reload diff --git a/prerm b/prerm new file mode 100644 index 000000000..6dd0e3ac4 --- /dev/null +++ b/prerm @@ -0,0 +1,4 @@ +#!/bin/bash + +systemctl stop pgcat +systemctl disable pgcat diff --git a/utilities/deb.sh b/utilities/deb.sh index a194d8038..b01a9929e 100644 --- a/utilities/deb.sh +++ b/utilities/deb.sh @@ -17,14 +17,21 @@ cargo build --release rm -rf "$deb_dir" mkdir -p "$deb_dir/DEBIAN" mkdir -p "$deb_dir/usr/bin" -mkdir -p "$deb_dir/etc" +mkdir -p "$deb_dir/etc/systemd/system" cp target/release/pgcat "$deb_dir/usr/bin/pgcat" chmod +x "$deb_dir/usr/bin/pgcat" cp pgcat.toml "$deb_dir/etc/pgcat.toml" +cp pgcat.service "$deb_dir/etc/systemd/system/pgcat.service" (cat control | envsubst) > "$deb_dir/DEBIAN/control" +cp postinst "$deb_dir/DEBIAN/postinst" +cp postrm "$deb_dir/DEBIAN/postrm" +cp prerm "$deb_dir/DEBIAN/prerm" + +chmod +x ${deb_dir}/DEBIAN/post* +chmod +x ${deb_dir}/DEBIAN/pre* dpkg-deb \ --root-owner-group \ From 73be80f9b59ed95d35787689c6cbdacab66dc2c3 Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Tue, 8 Aug 2023 16:10:03 -0400 Subject: [PATCH 446/551] Make infer role configurable and fix double parse bug (#533) * Make infer role configurable and fix double parse bug * Fix tests * Enable infer_role_from query in toml for tests * Fix test * Add max length config, add logging for which application is failing to parse, and change config name * fmt * Update src/config.rs --------- --- .circleci/pgcat.toml | 5 ++ examples/docker/pgcat.toml | 4 ++ pgcat.toml | 4 ++ src/client.rs | 112 +++++++++++++++++++---------- src/config.rs | 39 ++++++++++ src/pool.rs | 11 +++ src/query_router.rs | 81 ++++++++++++++------- tests/ruby/helpers/pgcat_helper.rb | 1 + 8 files changed, 197 insertions(+), 60 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 377680a0f..8b87aa038 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -74,6 +74,10 @@ default_role = "any" # we'll direct it to the primary. query_parser_enabled = true +# If the query parser is enabled and this setting is enabled, we'll attempt to +# infer the role from the query itself. +query_parser_read_write_splitting = true + # If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for # load balancing of read queries. Otherwise, the primary will only be used for write # queries. The primary can always be explicitely selected with our custom protocol. @@ -134,6 +138,7 @@ database = "shard2" pool_mode = "session" default_role = "primary" query_parser_enabled = true +query_parser_read_write_splitting = true primary_reads_enabled = true sharding_function = "pg_bigint_hash" diff --git a/examples/docker/pgcat.toml b/examples/docker/pgcat.toml index 5fd929de9..cfd94a1a8 100644 --- a/examples/docker/pgcat.toml +++ b/examples/docker/pgcat.toml @@ -71,6 +71,10 @@ default_role = "any" # we'll direct it to the primary. query_parser_enabled = true +# If the query parser is enabled and this setting is enabled, we'll attempt to +# infer the role from the query itself. +query_parser_read_write_splitting = true + # If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for # load balancing of read queries. Otherwise, the primary will only be used for write # queries. The primary can always be explicitly selected with our custom protocol. diff --git a/pgcat.toml b/pgcat.toml index d8ed1088b..ae5d74dce 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -162,6 +162,10 @@ default_role = "any" # we'll direct it to the primary. query_parser_enabled = true +# If the query parser is enabled and this setting is enabled, we'll attempt to +# infer the role from the query itself. +query_parser_read_write_splitting = true + # If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for # load balancing of read queries. Otherwise, the primary will only be used for write # queries. The primary can always be explicitly selected with our custom protocol. diff --git a/src/client.rs b/src/client.rs index 7d5e97987..4f5e6c96a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -774,6 +774,9 @@ where let mut prepared_statement = None; let mut will_prepare = false; + let client_identifier = + ClientIdentifier::new(&self.application_name, &self.username, &self.pool_name); + // Our custom protocol loop. // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocol. @@ -812,6 +815,21 @@ where message_result = read_message(&mut self.read) => message_result? }; + // Handle admin database queries. + if self.admin { + debug!("Handling admin command"); + handle_admin(&mut self.write, message, self.client_server_map.clone()).await?; + continue; + } + + // Get a pool instance referenced by the most up-to-date + // pointer. This ensures we always read the latest config + // when starting a query. + let mut pool = self.get_pool().await?; + query_router.update_pool_settings(pool.settings.clone()); + + let mut initial_parsed_ast = None; + match message[0] as char { // Buffer extended protocol messages even if we do not have // a server connection yet. Hopefully, when we get the S message @@ -841,24 +859,34 @@ where 'Q' => { if query_router.query_parser_enabled() { - if let Ok(ast) = QueryRouter::parse(&message) { - let plugin_result = query_router.execute_plugins(&ast).await; + match query_router.parse(&message) { + Ok(ast) => { + let plugin_result = query_router.execute_plugins(&ast).await; - match plugin_result { - Ok(PluginOutput::Deny(error)) => { - error_response(&mut self.write, &error).await?; - continue; - } + match plugin_result { + Ok(PluginOutput::Deny(error)) => { + error_response(&mut self.write, &error).await?; + continue; + } - Ok(PluginOutput::Intercept(result)) => { - write_all(&mut self.write, result).await?; - continue; - } + Ok(PluginOutput::Intercept(result)) => { + write_all(&mut self.write, result).await?; + continue; + } - _ => (), - }; + _ => (), + }; + + let _ = query_router.infer(&ast); - let _ = query_router.infer(&ast); + initial_parsed_ast = Some(ast); + } + Err(error) => { + warn!( + "Query parsing error: {} (client: {})", + error, client_identifier + ); + } } } } @@ -872,13 +900,21 @@ where self.buffer.put(&message[..]); if query_router.query_parser_enabled() { - if let Ok(ast) = QueryRouter::parse(&message) { - if let Ok(output) = query_router.execute_plugins(&ast).await { - plugin_output = Some(output); - } + match query_router.parse(&message) { + Ok(ast) => { + if let Ok(output) = query_router.execute_plugins(&ast).await { + plugin_output = Some(output); + } - let _ = query_router.infer(&ast); - } + let _ = query_router.infer(&ast); + } + Err(error) => { + warn!( + "Query parsing error: {} (client: {})", + error, client_identifier + ); + } + }; } continue; @@ -922,13 +958,6 @@ where _ => (), } - // Handle admin database queries. - if self.admin { - debug!("Handling admin command"); - handle_admin(&mut self.write, message, self.client_server_map.clone()).await?; - continue; - } - // Check on plugin results. match plugin_output { Some(PluginOutput::Deny(error)) => { @@ -941,11 +970,6 @@ where _ => (), }; - // Get a pool instance referenced by the most up-to-date - // pointer. This ensures we always read the latest config - // when starting a query. - let mut pool = self.get_pool().await?; - // Check if the pool is paused and wait until it's resumed. if pool.wait_paused().await { // Refresh pool information, something might have changed. @@ -1165,6 +1189,9 @@ where None => { trace!("Waiting for message inside transaction or in session mode"); + // This is not an initial message so discard the initial_parsed_ast + initial_parsed_ast.take(); + match tokio::time::timeout( idle_client_timeout_duration, read_message(&mut self.read), @@ -1221,7 +1248,22 @@ where // Query 'Q' => { if query_router.query_parser_enabled() { - if let Ok(ast) = QueryRouter::parse(&message) { + // We don't want to parse again if we already parsed it as the initial message + let ast = match initial_parsed_ast { + Some(_) => Some(initial_parsed_ast.take().unwrap()), + None => match query_router.parse(&message) { + Ok(ast) => Some(ast), + Err(error) => { + warn!( + "Query parsing error: {} (client: {})", + error, client_identifier + ); + None + } + }, + }; + + if let Some(ast) = ast { let plugin_result = query_router.execute_plugins(&ast).await; match plugin_result { @@ -1237,8 +1279,6 @@ where _ => (), }; - - let _ = query_router.infer(&ast); } } debug!("Sending query to server"); @@ -1290,7 +1330,7 @@ where } if query_router.query_parser_enabled() { - if let Ok(ast) = QueryRouter::parse(&message) { + if let Ok(ast) = query_router.parse(&message) { if let Ok(output) = query_router.execute_plugins(&ast).await { plugin_output = Some(output); } diff --git a/src/config.rs b/src/config.rs index 9228b9bbe..0e4b8c7d8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -511,6 +511,11 @@ pub struct Pool { #[serde(default)] // False pub query_parser_enabled: bool, + pub query_parser_max_length: Option, + + #[serde(default)] // False + pub query_parser_read_write_splitting: bool, + #[serde(default)] // False pub primary_reads_enabled: bool, @@ -627,6 +632,18 @@ impl Pool { } } + if self.query_parser_read_write_splitting && !self.query_parser_enabled { + error!( + "query_parser_read_write_splitting is only valid when query_parser_enabled is true" + ); + return Err(Error::BadConfig); + } + + if self.plugins.is_some() && !self.query_parser_enabled { + error!("plugins are only valid when query_parser_enabled is true"); + return Err(Error::BadConfig); + } + self.automatic_sharding_key = match &self.automatic_sharding_key { Some(key) => { // No quotes in the key so we don't have to compare quoted @@ -663,6 +680,8 @@ impl Default for Pool { users: BTreeMap::default(), default_role: String::from("any"), query_parser_enabled: false, + query_parser_max_length: None, + query_parser_read_write_splitting: false, primary_reads_enabled: false, sharding_function: ShardingFunction::PgBigintHash, automatic_sharding_key: None, @@ -914,6 +933,17 @@ impl From<&Config> for std::collections::HashMap { format!("pools.{}.query_parser_enabled", pool_name), pool.query_parser_enabled.to_string(), ), + ( + format!("pools.{}.query_parser_max_length", pool_name), + match pool.query_parser_max_length { + Some(max_length) => max_length.to_string(), + None => String::from("unlimited"), + }, + ), + ( + format!("pools.{}.query_parser_read_write_splitting", pool_name), + pool.query_parser_read_write_splitting.to_string(), + ), ( format!("pools.{}.default_role", pool_name), pool.default_role.clone(), @@ -1096,6 +1126,15 @@ impl Config { "[pool: {}] Query router: {}", pool_name, pool_config.query_parser_enabled ); + + info!( + "[pool: {}] Query parser max length: {:?}", + pool_name, pool_config.query_parser_max_length + ); + info!( + "[pool: {}] Infer role from query: {}", + pool_name, pool_config.query_parser_read_write_splitting + ); info!( "[pool: {}] Number of shards: {}", pool_name, diff --git a/src/pool.rs b/src/pool.rs index b92935217..dddb3ebe2 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -111,6 +111,12 @@ pub struct PoolSettings { // Enable/disable query parser. pub query_parser_enabled: bool, + // Max length of query the parser will parse. + pub query_parser_max_length: Option, + + // Infer role + pub query_parser_read_write_splitting: bool, + // Read from the primary as well or not. pub primary_reads_enabled: bool, @@ -157,6 +163,8 @@ impl Default for PoolSettings { db: String::default(), default_role: None, query_parser_enabled: false, + query_parser_max_length: None, + query_parser_read_write_splitting: false, primary_reads_enabled: true, sharding_function: ShardingFunction::PgBigintHash, automatic_sharding_key: None, @@ -456,6 +464,9 @@ impl ConnectionPool { _ => unreachable!(), }, query_parser_enabled: pool_config.query_parser_enabled, + query_parser_max_length: pool_config.query_parser_max_length, + query_parser_read_write_splitting: pool_config + .query_parser_read_write_splitting, primary_reads_enabled: pool_config.primary_reads_enabled, sharding_function: pool_config.sharding_function, automatic_sharding_key: pool_config.automatic_sharding_key.clone(), diff --git a/src/query_router.rs b/src/query_router.rs index 126b81380..9676a26f6 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -331,11 +331,23 @@ impl QueryRouter { Some((command, value)) } - pub fn parse(message: &BytesMut) -> Result, Error> { + pub fn parse(&self, message: &BytesMut) -> Result, Error> { let mut message_cursor = Cursor::new(message); let code = message_cursor.get_u8() as char; - let _len = message_cursor.get_i32() as usize; + let len = message_cursor.get_i32() as usize; + + match self.pool_settings.query_parser_max_length { + Some(max_length) => { + if len > max_length { + return Err(Error::QueryRouterParserError(format!( + "Query too long for parser: {} > {}", + len, max_length + ))); + } + } + None => (), + }; let query = match code { // Query @@ -372,6 +384,10 @@ impl QueryRouter { /// Try to infer which server to connect to based on the contents of the query. pub fn infer(&mut self, ast: &Vec) -> Result<(), Error> { + if !self.pool_settings.query_parser_read_write_splitting { + return Ok(()); // Nothing to do + } + debug!("Inferring role"); if ast.is_empty() { @@ -433,6 +449,10 @@ impl QueryRouter { /// N.B.: Only supports anonymous prepared statements since we don't /// keep a cache of them in PgCat. pub fn infer_shard_from_bind(&mut self, message: &BytesMut) -> bool { + if !self.pool_settings.query_parser_read_write_splitting { + return false; // Nothing to do + } + debug!("Parsing bind message"); let mut message_cursor = Cursor::new(message); @@ -910,6 +930,7 @@ mod test { fn test_infer_replica() { QueryRouter::setup(); let mut qr = QueryRouter::new(); + qr.pool_settings.query_parser_read_write_splitting = true; assert!(qr.try_execute_command(&simple_query("SET SERVER ROLE TO 'auto'")) != None); assert!(qr.query_parser_enabled()); @@ -925,7 +946,7 @@ mod test { for query in queries { // It's a recognized query - assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); + assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Replica)); } } @@ -934,6 +955,7 @@ mod test { fn test_infer_primary() { QueryRouter::setup(); let mut qr = QueryRouter::new(); + qr.pool_settings.query_parser_read_write_splitting = true; let queries = vec![ simple_query("UPDATE items SET name = 'pumpkin' WHERE id = 5"), @@ -944,7 +966,7 @@ mod test { for query in queries { // It's a recognized query - assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); + assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Primary)); } } @@ -956,7 +978,7 @@ mod test { let query = simple_query("SELECT * FROM items WHERE id = 5"); assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO on")) != None); - assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); + assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), None); } @@ -964,6 +986,8 @@ mod test { fn test_infer_parse_prepared() { QueryRouter::setup(); let mut qr = QueryRouter::new(); + qr.pool_settings.query_parser_read_write_splitting = true; + qr.try_execute_command(&simple_query("SET SERVER ROLE TO 'auto'")); assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO off")) != None); @@ -976,7 +1000,7 @@ mod test { res.put(prepared_stmt); res.put_i16(0); - assert!(qr.infer(&QueryRouter::parse(&res).unwrap()).is_ok()); + assert!(qr.infer(&qr.parse(&res).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Replica)); } @@ -1132,6 +1156,8 @@ mod test { fn test_enable_query_parser() { QueryRouter::setup(); let mut qr = QueryRouter::new(); + qr.pool_settings.query_parser_read_write_splitting = true; + let query = simple_query("SET SERVER ROLE TO 'auto'"); assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO off")) != None); @@ -1140,11 +1166,11 @@ mod test { assert_eq!(qr.role(), None); let query = simple_query("INSERT INTO test_table VALUES (1)"); - assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); + assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Primary)); let query = simple_query("SELECT * FROM test_table"); - assert!(qr.infer(&QueryRouter::parse(&query).unwrap()).is_ok()); + assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), Some(Role::Replica)); assert!(qr.query_parser_enabled()); @@ -1164,6 +1190,8 @@ mod test { user: crate::config::User::default(), default_role: Some(Role::Replica), query_parser_enabled: true, + query_parser_max_length: None, + query_parser_read_write_splitting: true, primary_reads_enabled: false, sharding_function: ShardingFunction::PgBigintHash, automatic_sharding_key: Some(String::from("test.id")), @@ -1208,18 +1236,18 @@ mod test { let mut qr = QueryRouter::new(); assert!(qr - .infer(&QueryRouter::parse(&simple_query("BEGIN; SELECT 1; COMMIT;")).unwrap()) + .infer(&qr.parse(&simple_query("BEGIN; SELECT 1; COMMIT;")).unwrap()) .is_ok()); assert_eq!(qr.role(), Role::Primary); assert!(qr - .infer(&QueryRouter::parse(&simple_query("SELECT 1; SELECT 2;")).unwrap()) + .infer(&qr.parse(&simple_query("SELECT 1; SELECT 2;")).unwrap()) .is_ok()); assert_eq!(qr.role(), Role::Replica); assert!(qr .infer( - &QueryRouter::parse(&simple_query( + &qr.parse(&simple_query( "SELECT 123; INSERT INTO t VALUES (5); SELECT 1;" )) .unwrap() @@ -1239,6 +1267,8 @@ mod test { user: crate::config::User::default(), default_role: Some(Role::Replica), query_parser_enabled: true, + query_parser_max_length: None, + query_parser_read_write_splitting: true, primary_reads_enabled: false, sharding_function: ShardingFunction::PgBigintHash, automatic_sharding_key: None, @@ -1284,15 +1314,19 @@ mod test { let mut qr = QueryRouter::new(); qr.pool_settings.automatic_sharding_key = Some("data.id".to_string()); qr.pool_settings.shards = 3; + qr.pool_settings.query_parser_read_write_splitting = true; assert!(qr - .infer(&QueryRouter::parse(&simple_query("SELECT * FROM data WHERE id = 5")).unwrap()) + .infer( + &qr.parse(&simple_query("SELECT * FROM data WHERE id = 5")) + .unwrap(), + ) .is_ok()); assert_eq!(qr.shard(), 2); assert!(qr .infer( - &QueryRouter::parse(&simple_query( + &qr.parse(&simple_query( "SELECT one, two, three FROM public.data WHERE id = 6" )) .unwrap() @@ -1302,7 +1336,7 @@ mod test { assert!(qr .infer( - &QueryRouter::parse(&simple_query( + &qr.parse(&simple_query( "SELECT * FROM data INNER JOIN t2 ON data.id = 5 AND t2.data_id = data.id @@ -1317,7 +1351,7 @@ mod test { // in the query. assert!(qr .infer( - &QueryRouter::parse(&simple_query( + &qr.parse(&simple_query( "SELECT * FROM t2 INNER JOIN data ON id = 6 AND data.id = t2.data_id" )) .unwrap() @@ -1327,7 +1361,7 @@ mod test { assert!(qr .infer( - &QueryRouter::parse(&simple_query( + &qr.parse(&simple_query( r#"SELECT * FROM "public"."data" WHERE "id" = 6"# )) .unwrap() @@ -1337,7 +1371,7 @@ mod test { assert!(qr .infer( - &QueryRouter::parse(&simple_query( + &qr.parse(&simple_query( r#"SELECT * FROM "public"."data" WHERE "data"."id" = 5"# )) .unwrap() @@ -1349,7 +1383,7 @@ mod test { qr.pool_settings.automatic_sharding_key = Some("*.unique_enough_column_name".to_string()); assert!(qr .infer( - &QueryRouter::parse(&simple_query( + &qr.parse(&simple_query( "SELECT * FROM table_x WHERE unique_enough_column_name = 6" )) .unwrap() @@ -1359,7 +1393,7 @@ mod test { assert!(qr .infer( - &QueryRouter::parse(&simple_query("SELECT * FROM table_y WHERE another_key = 5")) + &qr.parse(&simple_query("SELECT * FROM table_y WHERE another_key = 5")) .unwrap() ) .is_ok()); @@ -1385,10 +1419,9 @@ mod test { let mut qr = QueryRouter::new(); qr.pool_settings.automatic_sharding_key = Some("data.id".to_string()); qr.pool_settings.shards = 3; + qr.pool_settings.query_parser_read_write_splitting = true; - assert!(qr - .infer(&QueryRouter::parse(&simple_query(stmt)).unwrap()) - .is_ok()); + assert!(qr.infer(&qr.parse(&simple_query(stmt)).unwrap()).is_ok()); assert_eq!(qr.placeholders.len(), 1); assert!(qr.infer_shard_from_bind(&bind)); @@ -1419,7 +1452,7 @@ mod test { qr.update_pool_settings(pool_settings); let query = simple_query("SELECT * FROM pg_database"); - let ast = QueryRouter::parse(&query).unwrap(); + let ast = qr.parse(&query).unwrap(); let res = qr.execute_plugins(&ast).await; @@ -1437,7 +1470,7 @@ mod test { let qr = QueryRouter::new(); let query = simple_query("SELECT * FROM pg_database"); - let ast = QueryRouter::parse(&query).unwrap(); + let ast = qr.parse(&query).unwrap(); let res = qr.execute_plugins(&ast).await; diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index 7a5bd71d6..9b764d877 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -34,6 +34,7 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mod "load_balancing_mode" => lb_mode, "primary_reads_enabled" => true, "query_parser_enabled" => true, + "query_parser_read_write_splitting" => true, "automatic_sharding_key" => "data.id", "sharding_function" => "pg_bigint_hash", "shards" => { From 53d24cb2c48b45f29e768b3c8f0c59939e6558f0 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Aug 2023 13:48:52 -0700 Subject: [PATCH 447/551] Preserve existing behavior (#541) --- src/config.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 0e4b8c7d8..803d3a755 100644 --- a/src/config.rs +++ b/src/config.rs @@ -513,7 +513,7 @@ pub struct Pool { pub query_parser_max_length: Option, - #[serde(default)] // False + #[serde(default = "Pool::query_parser_read_write_splitting")] // False pub query_parser_read_write_splitting: bool, #[serde(default)] // False @@ -592,6 +592,10 @@ impl Pool { true } + pub fn query_parser_read_write_splitting() -> bool { + true + } + pub fn validate(&mut self) -> Result<(), Error> { match self.default_role.as_ref() { "any" => (), From cd8656f38f34dc88e2f9b36c4e55db26a72bfb93 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 8 Aug 2023 17:45:48 -0700 Subject: [PATCH 448/551] Revert "Preserve existing behavior" (#542) Revert "Preserve existing behavior (#541)" This reverts commit 53d24cb2c48b45f29e768b3c8f0c59939e6558f0. --- src/config.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/config.rs b/src/config.rs index 803d3a755..0e4b8c7d8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -513,7 +513,7 @@ pub struct Pool { pub query_parser_max_length: Option, - #[serde(default = "Pool::query_parser_read_write_splitting")] // False + #[serde(default)] // False pub query_parser_read_write_splitting: bool, #[serde(default)] // False @@ -592,10 +592,6 @@ impl Pool { true } - pub fn query_parser_read_write_splitting() -> bool { - true - } - pub fn validate(&mut self) -> Result<(), Error> { match self.default_role.as_ref() { "any" => (), From ae1477ebd686c498e5ea5fa67cff2c07b0968010 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 9 Aug 2023 13:14:05 -0300 Subject: [PATCH 449/551] parse server error messages (#543) This commit adds a parser to the Postgres error message, providing better error messages. Implemented based in: https://www.postgresql.org/docs/12/protocol-error-fields.html Signed-off-by: Sebastian Webber --- src/errors.rs | 1 + src/messages.rs | 298 ++++++++++++++++++++++++++++++++++++++++++++++++ src/server.rs | 15 ++- 3 files changed, 308 insertions(+), 6 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index c076a3f93..014a1340d 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -12,6 +12,7 @@ pub enum Error { ProtocolSyncError(String), BadQuery(String), ServerError, + ServerMessageParserError(String), ServerStartupError(String, ServerIdentifier), ServerAuthError(String, ServerIdentifier), BadConfig, diff --git a/src/messages.rs b/src/messages.rs index 8ebc00a34..1f40f1df2 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -11,10 +11,13 @@ use crate::client::PREPARED_STATEMENT_COUNTER; use crate::config::get_config; use crate::errors::Error; +use crate::constants::MESSAGE_TERMINATOR; use std::collections::HashMap; use std::ffi::CString; +use std::fmt::{Display, Formatter}; use std::io::{BufRead, Cursor}; use std::mem; +use std::str::FromStr; use std::sync::atomic::Ordering; use std::time::Duration; @@ -1098,3 +1101,298 @@ pub fn prepared_statement_name() -> String { PREPARED_STATEMENT_COUNTER.fetch_add(1, Ordering::SeqCst) ) } + +// from https://www.postgresql.org/docs/12/protocol-error-fields.html +#[derive(Debug, Default, PartialEq)] +pub struct PgErrorMsg { + pub severity_localized: String, // S + pub severity: String, // V + pub code: String, // C + pub message: String, // M + pub detail: Option, // D + pub hint: Option, // H + pub position: Option, // P + pub internal_position: Option, // p + pub internal_query: Option, // q + pub where_context: Option, // W + pub schema_name: Option, // s + pub table_name: Option, // t + pub column_name: Option, // c + pub data_type_name: Option, // d + pub constraint_name: Option, // n + pub file_name: Option, // F + pub line: Option, // L + pub routine: Option, // R +} + +// TODO: implement with https://docs.rs/derive_more/latest/derive_more/ +impl Display for PgErrorMsg { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "[severity: {}]", self.severity)?; + write!(f, "[code: {}]", self.code)?; + write!(f, "[message: {}]", self.message)?; + if let Some(val) = &self.detail { + write!(f, "[detail: {val}]")?; + } + if let Some(val) = &self.hint { + write!(f, "[hint: {val}]")?; + } + if let Some(val) = &self.position { + write!(f, "[position: {val}]")?; + } + if let Some(val) = &self.internal_position { + write!(f, "[internal_position: {val}]")?; + } + if let Some(val) = &self.internal_query { + write!(f, "[internal_query: {val}]")?; + } + if let Some(val) = &self.internal_query { + write!(f, "[internal_query: {val}]")?; + } + if let Some(val) = &self.where_context { + write!(f, "[where: {val}]")?; + } + if let Some(val) = &self.schema_name { + write!(f, "[schema_name: {val}]")?; + } + if let Some(val) = &self.table_name { + write!(f, "[table_name: {val}]")?; + } + if let Some(val) = &self.column_name { + write!(f, "[column_name: {val}]")?; + } + if let Some(val) = &self.data_type_name { + write!(f, "[data_type_name: {val}]")?; + } + if let Some(val) = &self.constraint_name { + write!(f, "[constraint_name: {val}]")?; + } + if let Some(val) = &self.file_name { + write!(f, "[file_name: {val}]")?; + } + if let Some(val) = &self.line { + write!(f, "[line: {val}]")?; + } + if let Some(val) = &self.routine { + write!(f, "[routine: {val}]")?; + } + + write!(f, " ")?; + + Ok(()) + } +} + +impl PgErrorMsg { + pub fn parse(error_msg: Vec) -> Result { + let mut out = PgErrorMsg { + severity_localized: "".to_string(), + severity: "".to_string(), + code: "".to_string(), + message: "".to_string(), + detail: None, + hint: None, + position: None, + internal_position: None, + internal_query: None, + where_context: None, + schema_name: None, + table_name: None, + column_name: None, + data_type_name: None, + constraint_name: None, + file_name: None, + line: None, + routine: None, + }; + for msg_part in error_msg.split(|v| *v == MESSAGE_TERMINATOR) { + if msg_part.is_empty() { + continue; + } + + let msg_content = match String::from_utf8_lossy(&msg_part[1..]).parse() { + Ok(c) => c, + Err(err) => { + return Err(Error::ServerMessageParserError(format!( + "could not parse server message field. err {:?}", + err + ))) + } + }; + + match &msg_part[0] { + b'S' => { + out.severity_localized = msg_content; + } + b'V' => { + out.severity = msg_content; + } + b'C' => { + out.code = msg_content; + } + b'M' => { + out.message = msg_content; + } + b'D' => { + out.detail = Some(msg_content); + } + b'H' => { + out.hint = Some(msg_content); + } + b'P' => out.position = Some(u32::from_str(msg_content.as_str()).unwrap_or(0)), + b'p' => { + out.internal_position = Some(u32::from_str(msg_content.as_str()).unwrap_or(0)) + } + b'q' => { + out.internal_query = Some(msg_content); + } + b'W' => { + out.where_context = Some(msg_content); + } + b's' => { + out.schema_name = Some(msg_content); + } + b't' => { + out.table_name = Some(msg_content); + } + b'c' => { + out.column_name = Some(msg_content); + } + b'd' => { + out.data_type_name = Some(msg_content); + } + b'n' => { + out.constraint_name = Some(msg_content); + } + b'F' => { + out.file_name = Some(msg_content); + } + b'L' => out.line = Some(u32::from_str(msg_content.as_str()).unwrap_or(0)), + b'R' => { + out.routine = Some(msg_content); + } + _ => {} + } + } + + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use crate::messages::PgErrorMsg; + use log::{error, info}; + + fn field(kind: char, content: &str) -> Vec { + format!("{kind}{content}\0").as_bytes().to_vec() + } + + #[test] + fn parse_fields() { + let mut complete_msg = vec![]; + let severity = "FATAL"; + complete_msg.extend(field('S', &severity)); + complete_msg.extend(field('V', &severity)); + + let error_code = "29P02"; + complete_msg.extend(field('C', &error_code)); + let message = "password authentication failed for user \"wrong_user\""; + complete_msg.extend(field('M', &message)); + let detail_msg = "super detailed message"; + complete_msg.extend(field('D', &detail_msg)); + let hint_msg = "hint detail here"; + complete_msg.extend(field('H', &hint_msg)); + complete_msg.extend(field('P', "123")); + complete_msg.extend(field('p', "234")); + let internal_query = "SELECT * from foo;"; + complete_msg.extend(field('q', &internal_query)); + let where_msg = "where goes here"; + complete_msg.extend(field('W', &where_msg)); + let schema_msg = "schema_name"; + complete_msg.extend(field('s', &schema_msg)); + let table_msg = "table_name"; + complete_msg.extend(field('t', &table_msg)); + let column_msg = "column_name"; + complete_msg.extend(field('c', &column_msg)); + let data_type_msg = "type_name"; + complete_msg.extend(field('d', &data_type_msg)); + let constraint_msg = "constraint_name"; + complete_msg.extend(field('n', &constraint_msg)); + let file_msg = "pgcat.c"; + complete_msg.extend(field('F', &file_msg)); + complete_msg.extend(field('L', "335")); + let routine_msg = "my_failing_routine"; + complete_msg.extend(field('R', &routine_msg)); + + tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_ansi(true) + .init(); + + info!( + "full message: {}", + PgErrorMsg::parse(complete_msg.clone()).unwrap() + ); + assert_eq!( + PgErrorMsg { + severity_localized: severity.to_string(), + severity: severity.to_string(), + code: error_code.to_string(), + message: message.to_string(), + detail: Some(detail_msg.to_string()), + hint: Some(hint_msg.to_string()), + position: Some(123), + internal_position: Some(234), + internal_query: Some(internal_query.to_string()), + where_context: Some(where_msg.to_string()), + schema_name: Some(schema_msg.to_string()), + table_name: Some(table_msg.to_string()), + column_name: Some(column_msg.to_string()), + data_type_name: Some(data_type_msg.to_string()), + constraint_name: Some(constraint_msg.to_string()), + file_name: Some(file_msg.to_string()), + line: Some(335), + routine: Some(routine_msg.to_string()), + }, + PgErrorMsg::parse(complete_msg).unwrap() + ); + + let mut only_mandatory_msg = vec![]; + only_mandatory_msg.extend(field('S', &severity)); + only_mandatory_msg.extend(field('V', &severity)); + only_mandatory_msg.extend(field('C', &error_code)); + only_mandatory_msg.extend(field('M', &message)); + only_mandatory_msg.extend(field('D', &detail_msg)); + + let err_fields = PgErrorMsg::parse(only_mandatory_msg.clone()).unwrap(); + info!("only mandatory fields: {}", &err_fields); + error!( + "server error: {}: {}", + err_fields.severity, err_fields.message + ); + assert_eq!( + PgErrorMsg { + severity_localized: severity.to_string(), + severity: severity.to_string(), + code: error_code.to_string(), + message: message.to_string(), + detail: Some(detail_msg.to_string()), + hint: None, + position: None, + internal_position: None, + internal_query: None, + where_context: None, + schema_name: None, + table_name: None, + column_name: None, + data_type_name: None, + constraint_name: None, + file_name: None, + line: None, + routine: None, + }, + PgErrorMsg::parse(only_mandatory_msg).unwrap() + ); + } +} diff --git a/src/server.rs b/src/server.rs index afa1c09de..9d0beaace 100644 --- a/src/server.rs +++ b/src/server.rs @@ -588,8 +588,7 @@ impl Server { // An error message will be present. _ => { - // Read the error message without the terminating null character. - let mut error = vec![0u8; len as usize - 4 - 1]; + let mut error = vec![0u8; len as usize]; match stream.read_exact(&mut error).await { Ok(_) => (), @@ -601,10 +600,14 @@ impl Server { } }; - // TODO: the error message contains multiple fields; we can decode them and - // present a prettier message to the user. - // See: https://www.postgresql.org/docs/12/protocol-error-fields.html - error!("Server error: {}", String::from_utf8_lossy(&error)); + let fields = match PgErrorMsg::parse(error) { + Ok(f) => f, + Err(err) => { + return Err(err); + } + }; + trace!("error fields: {}", &fields); + error!("server error: {}: {}", fields.severity, fields.message); } }; From 46a10d07def41d46cdaf616d4105e0378520b5b9 Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 10 Aug 2023 11:18:46 -0400 Subject: [PATCH 450/551] Handle and track startup parameters (#478) * User server parameters struct instead of server info bytesmut * Refactor to use hashmap for all params and add server parameters to client * Sync parameters on client server checkout * minor refactor * update client side parameters when changed * Move the SET statement logic from the C packet to the S packet. * trigger build * revert validation changes * remove comment * Try fix * Reset cleanup state after sync * fix server version test * Track application name through client life for stats * Add tests * minor refactoring * fmt * fix * fmt --- src/admin.rs | 17 ++- src/client.rs | 107 ++++++------- src/messages.rs | 19 +++ src/mirrors.rs | 2 +- src/pool.rs | 26 ++-- src/server.rs | 226 ++++++++++++++++++++++------ tests/ruby/helpers/pgcat_process.rb | 10 +- tests/ruby/misc_spec.rb | 24 +++ 8 files changed, 308 insertions(+), 123 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 6c83f9b20..f27b2a0d1 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -1,4 +1,5 @@ use crate::pool::BanReason; +use crate::server::ServerParameters; use crate::stats::pool::PoolStats; use bytes::{Buf, BufMut, BytesMut}; use log::{error, info, trace}; @@ -17,16 +18,16 @@ use crate::pool::ClientServerMap; use crate::pool::{get_all_pools, get_pool}; use crate::stats::{get_client_stats, get_server_stats, ClientState, ServerState}; -pub fn generate_server_info_for_admin() -> BytesMut { - let mut server_info = BytesMut::new(); +pub fn generate_server_parameters_for_admin() -> ServerParameters { + let mut server_parameters = ServerParameters::new(); - server_info.put(server_parameter_message("application_name", "")); - server_info.put(server_parameter_message("client_encoding", "UTF8")); - server_info.put(server_parameter_message("server_encoding", "UTF8")); - server_info.put(server_parameter_message("server_version", VERSION)); - server_info.put(server_parameter_message("DateStyle", "ISO, MDY")); + server_parameters.set_param("application_name".to_string(), "".to_string(), true); + server_parameters.set_param("client_encoding".to_string(), "UTF8".to_string(), true); + server_parameters.set_param("server_encoding".to_string(), "UTF8".to_string(), true); + server_parameters.set_param("server_version".to_string(), VERSION.to_string(), true); + server_parameters.set_param("DateStyle".to_string(), "ISO, MDY".to_string(), true); - server_info + server_parameters } /// Handle admin client. diff --git a/src/client.rs b/src/client.rs index 4f5e6c96a..6cdea9878 100644 --- a/src/client.rs +++ b/src/client.rs @@ -12,7 +12,7 @@ use tokio::net::TcpStream; use tokio::sync::broadcast::Receiver; use tokio::sync::mpsc::Sender; -use crate::admin::{generate_server_info_for_admin, handle_admin}; +use crate::admin::{generate_server_parameters_for_admin, handle_admin}; use crate::auth_passthrough::refetch_auth_hash; use crate::config::{ get_config, get_idle_client_in_transaction_timeout, get_prepared_statements, Address, PoolMode, @@ -22,7 +22,7 @@ use crate::messages::*; use crate::plugins::PluginOutput; use crate::pool::{get_pool, ClientServerMap, ConnectionPool}; use crate::query_router::{Command, QueryRouter}; -use crate::server::Server; +use crate::server::{Server, ServerParameters}; use crate::stats::{ClientStats, ServerStats}; use crate::tls::Tls; @@ -96,8 +96,8 @@ pub struct Client { /// Postgres user for this client (This comes from the user in the connection string) username: String, - /// Application name for this client (defaults to pgcat) - application_name: String, + /// Server startup and session parameters that we're going to track + server_parameters: ServerParameters, /// Used to notify clients about an impending shutdown shutdown: Receiver<()>, @@ -502,7 +502,7 @@ where }; // Authenticate admin user. - let (transaction_mode, server_info) = if admin { + let (transaction_mode, mut server_parameters) = if admin { let config = get_config(); // Compare server and client hashes. @@ -521,7 +521,7 @@ where return Err(error); } - (false, generate_server_info_for_admin()) + (false, generate_server_parameters_for_admin()) } // Authenticate normal user. else { @@ -654,13 +654,16 @@ where } } - (transaction_mode, pool.server_info()) + (transaction_mode, pool.server_parameters()) }; + // Update the parameters to merge what the application sent and what's originally on the server + server_parameters.set_from_hashmap(¶meters, false); + debug!("Password authentication successful"); auth_ok(&mut write).await?; - write_all(&mut write, server_info).await?; + write_all(&mut write, (&server_parameters).into()).await?; backend_key_data(&mut write, process_id, secret_key).await?; ready_for_query(&mut write).await?; @@ -690,7 +693,7 @@ where last_server_stats: None, pool_name: pool_name.clone(), username: username.clone(), - application_name: application_name.to_string(), + server_parameters, shutdown, connected_to_server: false, prepared_statements: HashMap::new(), @@ -725,7 +728,7 @@ where last_server_stats: None, pool_name: String::from("undefined"), username: String::from("undefined"), - application_name: String::from("undefined"), + server_parameters: ServerParameters::new(), shutdown, connected_to_server: false, prepared_statements: HashMap::new(), @@ -774,8 +777,11 @@ where let mut prepared_statement = None; let mut will_prepare = false; - let client_identifier = - ClientIdentifier::new(&self.application_name, &self.username, &self.pool_name); + let client_identifier = ClientIdentifier::new( + &self.server_parameters.get_application_name(), + &self.username, + &self.pool_name, + ); // Our custom protocol loop. // We expect the client to either start a transaction with regular queries @@ -1115,10 +1121,7 @@ where server.address() ); - // TODO: investigate other parameters and set them too. - - // Set application_name. - server.set_name(&self.application_name).await?; + server.sync_parameters(&self.server_parameters).await?; let mut initial_message = Some(message); @@ -1296,7 +1299,9 @@ where if !server.in_transaction() { // Report transaction executed statistics. self.stats.transaction(); - server.stats().transaction(&self.application_name); + server + .stats() + .transaction(&self.server_parameters.get_application_name()); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -1446,7 +1451,9 @@ where if !server.in_transaction() { self.stats.transaction(); - server.stats().transaction(&self.application_name); + server + .stats() + .transaction(&self.server_parameters.get_application_name()); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -1495,7 +1502,9 @@ where if !server.in_transaction() { self.stats.transaction(); - server.stats().transaction(&self.application_name); + server + .stats() + .transaction(self.server_parameters.get_application_name()); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -1547,7 +1556,9 @@ where Err(Error::ClientError(format!( "Invalid pool name {{ username: {}, pool_name: {}, application_name: {} }}", - self.pool_name, self.username, self.application_name + self.pool_name, + self.username, + self.server_parameters.get_application_name() ))) } } @@ -1704,7 +1715,7 @@ where client_stats.query(); server.stats().query( Instant::now().duration_since(query_start).as_millis() as u64, - &self.application_name, + &self.server_parameters.get_application_name(), ); Ok(()) @@ -1733,38 +1744,18 @@ where pool: &ConnectionPool, client_stats: &ClientStats, ) -> Result { - if pool.settings.user.statement_timeout > 0 { - match tokio::time::timeout( - tokio::time::Duration::from_millis(pool.settings.user.statement_timeout), - server.recv(), - ) - .await - { - Ok(result) => match result { - Ok(message) => Ok(message), - Err(err) => { - pool.ban(address, BanReason::MessageReceiveFailed, Some(client_stats)); - error_response_terminal( - &mut self.write, - &format!("error receiving data from server: {:?}", err), - ) - .await?; - Err(err) - } - }, - Err(_) => { - error!( - "Statement timeout while talking to {:?} with user {}", - address, pool.settings.user.username - ); - server.mark_bad(); - pool.ban(address, BanReason::StatementTimeout, Some(client_stats)); - error_response_terminal(&mut self.write, "pool statement timeout").await?; - Err(Error::StatementTimeout) - } - } - } else { - match server.recv().await { + let statement_timeout_duration = match pool.settings.user.statement_timeout { + 0 => tokio::time::Duration::MAX, + timeout => tokio::time::Duration::from_millis(timeout), + }; + + match tokio::time::timeout( + statement_timeout_duration, + server.recv(Some(&mut self.server_parameters)), + ) + .await + { + Ok(result) => match result { Ok(message) => Ok(message), Err(err) => { pool.ban(address, BanReason::MessageReceiveFailed, Some(client_stats)); @@ -1775,6 +1766,16 @@ where .await?; Err(err) } + }, + Err(_) => { + error!( + "Statement timeout while talking to {:?} with user {}", + address, pool.settings.user.username + ); + server.mark_bad(); + pool.ban(address, BanReason::StatementTimeout, Some(client_stats)); + error_response_terminal(&mut self.write, "pool statement timeout").await?; + Err(Error::StatementTimeout) } } } diff --git a/src/messages.rs b/src/messages.rs index 1f40f1df2..07fe93176 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -144,6 +144,10 @@ where bytes.put_slice(user.as_bytes()); bytes.put_u8(0); + // Application name + bytes.put(&b"application_name\0"[..]); + bytes.put_slice(&b"pgcat\0"[..]); + // Database bytes.put(&b"database\0"[..]); bytes.put_slice(database.as_bytes()); @@ -731,6 +735,21 @@ impl BytesMutReader for Cursor<&BytesMut> { } } +impl BytesMutReader for BytesMut { + /// Should only be used when reading strings from the message protocol. + /// Can be used to read multiple strings from the same message which are separated by the null byte + fn read_string(&mut self) -> Result { + let null_index = self.iter().position(|&byte| byte == b'\0'); + + match null_index { + Some(index) => { + let string_bytes = self.split_to(index + 1); + Ok(String::from_utf8_lossy(&string_bytes[..string_bytes.len() - 1]).to_string()) + } + None => return Err(Error::ParseBytesError("Could not read string".to_string())), + } + } +} /// Parse (F) message. /// See: #[derive(Clone, Debug)] diff --git a/src/mirrors.rs b/src/mirrors.rs index 0f2b02c01..7922e6f88 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -78,7 +78,7 @@ impl MirroredClient { } // Incoming data from server (we read to clear the socket buffer and discard the data) - recv_result = server.recv() => { + recv_result = server.recv(None) => { match recv_result { Ok(message) => trace!("Received from mirror: {} {:?}", String::from_utf8_lossy(&message[..]), address.clone()), Err(err) => { diff --git a/src/pool.rs b/src/pool.rs index dddb3ebe2..b3627448a 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,7 +1,6 @@ use arc_swap::ArcSwap; use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection, QueueStrategy}; -use bytes::{BufMut, BytesMut}; use chrono::naive::NaiveDateTime; use log::{debug, error, info, warn}; use once_cell::sync::Lazy; @@ -25,7 +24,7 @@ use crate::errors::Error; use crate::auth_passthrough::AuthPassthrough; use crate::plugins::prewarmer; -use crate::server::Server; +use crate::server::{Server, ServerParameters}; use crate::sharding::ShardingFunction; use crate::stats::{AddressStats, ClientStats, ServerStats}; @@ -196,10 +195,10 @@ pub struct ConnectionPool { /// that should not be queried. banlist: BanList, - /// The server information (K messages) have to be passed to the + /// The server information has to be passed to the /// clients on startup. We pre-connect to all shards and replicas - /// on pool creation and save the K messages here. - server_info: Arc>, + /// on pool creation and save the startup parameters here. + original_server_parameters: Arc>, /// Pool configuration. pub settings: PoolSettings, @@ -445,7 +444,7 @@ impl ConnectionPool { addresses, banlist: Arc::new(RwLock::new(banlist)), config_hash: new_pool_hash_value, - server_info: Arc::new(RwLock::new(BytesMut::new())), + original_server_parameters: Arc::new(RwLock::new(ServerParameters::new())), auth_hash: pool_auth_hash, settings: PoolSettings { pool_mode: match user.pool_mode { @@ -528,7 +527,7 @@ impl ConnectionPool { for server in 0..self.servers(shard) { let databases = self.databases.clone(); let validated = Arc::clone(&validated); - let pool_server_info = Arc::clone(&self.server_info); + let pool_server_parameters = Arc::clone(&self.original_server_parameters); let task = tokio::task::spawn(async move { let connection = match databases[shard][server].get().await { @@ -541,11 +540,10 @@ impl ConnectionPool { let proxy = connection; let server = &*proxy; - let server_info = server.server_info(); + let server_parameters: ServerParameters = server.server_parameters(); - let mut guard = pool_server_info.write(); - guard.clear(); - guard.put(server_info.clone()); + let mut guard = pool_server_parameters.write(); + *guard = server_parameters; validated.store(true, Ordering::Relaxed); }); @@ -557,7 +555,7 @@ impl ConnectionPool { // TODO: compare server information to make sure // all shards are running identical configurations. - if self.server_info.read().is_empty() { + if !self.validated() { error!("Could not validate connection pool"); return Err(Error::AllServersDown); } @@ -917,8 +915,8 @@ impl ConnectionPool { &self.addresses[shard][server] } - pub fn server_info(&self) -> BytesMut { - self.server_info.read().clone() + pub fn server_parameters(&self) -> ServerParameters { + self.original_server_parameters.read().clone() } fn busy_connection_count(&self, address: &Address) -> u32 { diff --git a/src/server.rs b/src/server.rs index 9d0beaace..c4d7a1af4 100644 --- a/src/server.rs +++ b/src/server.rs @@ -3,12 +3,13 @@ use bytes::{Buf, BufMut, BytesMut}; use fallible_iterator::FallibleIterator; use log::{debug, error, info, trace, warn}; +use once_cell::sync::Lazy; use parking_lot::{Mutex, RwLock}; use postgres_protocol::message; -use std::collections::{BTreeSet, HashMap}; -use std::io::Read; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::mem; use std::net::IpAddr; -use std::sync::Arc; +use std::sync::{Arc, Once}; use std::time::SystemTime; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, BufStream}; use tokio::net::TcpStream; @@ -19,6 +20,7 @@ use crate::config::{get_config, get_prepared_statements_cache_size, Address, Use use crate::constants::*; use crate::dns_cache::{AddrSet, CACHED_RESOLVER}; use crate::errors::{Error, ServerIdentifier}; +use crate::messages::BytesMutReader; use crate::messages::*; use crate::mirrors::MirroringManager; use crate::pool::ClientServerMap; @@ -145,6 +147,124 @@ impl std::fmt::Display for CleanupState { } } +static TRACKED_PARAMETERS: Lazy> = Lazy::new(|| { + let mut set = HashSet::new(); + set.insert("client_encoding".to_string()); + set.insert("DateStyle".to_string()); + set.insert("TimeZone".to_string()); + set.insert("standard_conforming_strings".to_string()); + set.insert("application_name".to_string()); + set +}); + +#[derive(Debug, Clone)] +pub struct ServerParameters { + parameters: HashMap, +} + +impl Default for ServerParameters { + fn default() -> Self { + Self::new() + } +} + +impl ServerParameters { + pub fn new() -> Self { + let mut server_parameters = ServerParameters { + parameters: HashMap::new(), + }; + + server_parameters.set_param("client_encoding".to_string(), "UTF8".to_string(), false); + server_parameters.set_param("DateStyle".to_string(), "ISO, MDY".to_string(), false); + server_parameters.set_param("TimeZone".to_string(), "Etc/UTC".to_string(), false); + server_parameters.set_param( + "standard_conforming_strings".to_string(), + "on".to_string(), + false, + ); + server_parameters.set_param("application_name".to_string(), "pgcat".to_string(), false); + + server_parameters + } + + /// returns true if a tracked parameter was set, false if it was a non-tracked parameter + /// if startup is false, then then only tracked parameters will be set + pub fn set_param(&mut self, mut key: String, value: String, startup: bool) { + // The startup parameter will send uncapitalized keys but parameter status packets will send capitalized keys + if key == "timezone" { + key = "TimeZone".to_string(); + } else if key == "datestyle" { + key = "DateStyle".to_string(); + }; + + if TRACKED_PARAMETERS.contains(&key) { + self.parameters.insert(key, value); + } else { + if startup { + self.parameters.insert(key, value); + } + } + } + + pub fn set_from_hashmap(&mut self, parameters: &HashMap, startup: bool) { + // iterate through each and call set_param + for (key, value) in parameters { + self.set_param(key.to_string(), value.to_string(), startup); + } + } + + // Gets the diff of the parameters + fn compare_params(&self, incoming_parameters: &ServerParameters) -> HashMap { + let mut diff = HashMap::new(); + + // iterate through tracked parameters + for key in TRACKED_PARAMETERS.iter() { + if let Some(incoming_value) = incoming_parameters.parameters.get(key) { + if let Some(value) = self.parameters.get(key) { + if value != incoming_value { + diff.insert(key.to_string(), incoming_value.to_string()); + } + } + } + } + + diff + } + + pub fn get_application_name(&self) -> &String { + // Can unwrap because we set it in the constructor + self.parameters.get("application_name").unwrap() + } + + fn add_parameter_message(key: &str, value: &str, buffer: &mut BytesMut) { + buffer.put_u8(b'S'); + + // 4 is len of i32, the plus for the null terminator + let len = 4 + key.len() + 1 + value.len() + 1; + + buffer.put_i32(len as i32); + + buffer.put_slice(key.as_bytes()); + buffer.put_u8(0); + buffer.put_slice(value.as_bytes()); + buffer.put_u8(0); + } +} + +impl From<&ServerParameters> for BytesMut { + fn from(server_parameters: &ServerParameters) -> Self { + let mut bytes = BytesMut::new(); + + for (key, value) in &server_parameters.parameters { + ServerParameters::add_parameter_message(key, value, &mut bytes); + } + + bytes + } +} + +// pub fn compare + /// Server state. pub struct Server { /// Server host, e.g. localhost, @@ -158,7 +278,7 @@ pub struct Server { buffer: BytesMut, /// Server information the server sent us over on startup. - server_info: BytesMut, + server_parameters: ServerParameters, /// Backend id and secret key used for query cancellation. process_id: i32, @@ -347,7 +467,6 @@ impl Server { startup(&mut stream, username, database).await?; - let mut server_info = BytesMut::new(); let mut process_id: i32 = 0; let mut secret_key: i32 = 0; let server_identifier = ServerIdentifier::new(username, &database); @@ -359,6 +478,8 @@ impl Server { None => None, }; + let mut server_parameters = ServerParameters::new(); + loop { let code = match stream.read_u8().await { Ok(code) => code as char, @@ -616,9 +737,10 @@ impl Server { // ParameterStatus 'S' => { - let mut param = vec![0u8; len as usize - 4]; + let mut bytes = BytesMut::with_capacity(len as usize - 4); + bytes.resize(len as usize - mem::size_of::(), b'0'); - match stream.read_exact(&mut param).await { + match stream.read_exact(&mut bytes[..]).await { Ok(_) => (), Err(_) => { return Err(Error::ServerStartupError( @@ -628,12 +750,13 @@ impl Server { } }; + let key = bytes.read_string().unwrap(); + let value = bytes.read_string().unwrap(); + // Save the parameter so we can pass it to the client later. // These can be server_encoding, client_encoding, server timezone, Postgres version, // and many more interesting things we should know about the Postgres server we are talking to. - server_info.put_u8(b'S'); - server_info.put_i32(len); - server_info.put_slice(¶m[..]); + server_parameters.set_param(key, value, true); } // BackendKeyData @@ -675,11 +798,11 @@ impl Server { } }; - let mut server = Server { + let server = Server { address: address.clone(), stream: BufStream::new(stream), buffer: BytesMut::with_capacity(8196), - server_info, + server_parameters, process_id, secret_key, in_transaction: false, @@ -691,7 +814,7 @@ impl Server { addr_set, connected_at: chrono::offset::Utc::now().naive_utc(), stats, - application_name: String::new(), + application_name: "pgcat".to_string(), last_activity: SystemTime::now(), mirror_manager: match address.mirrors.len() { 0 => None, @@ -705,8 +828,6 @@ impl Server { prepared_statements: BTreeSet::new(), }; - server.set_name("pgcat").await?; - return Ok(server); } @@ -776,7 +897,10 @@ impl Server { /// Receive data from the server in response to a client request. /// This method must be called multiple times while `self.is_data_available()` is true /// in order to receive all data the server has to offer. - pub async fn recv(&mut self) -> Result { + pub async fn recv( + &mut self, + mut client_server_parameters: Option<&mut ServerParameters>, + ) -> Result { loop { let mut message = match read_message(&mut self.stream).await { Ok(message) => message, @@ -848,14 +972,13 @@ impl Server { self.in_copy_mode = false; } - let mut command_tag = String::new(); - match message.reader().read_to_string(&mut command_tag) { - Ok(_) => { + match message.read_string() { + Ok(command) => { // Non-exhaustive list of commands that are likely to change session variables/resources // which can leak between clients. This is a best effort to block bad clients // from poisoning a transaction-mode pool by setting inappropriate session variables - match command_tag.as_str() { - "SET\0" => { + match command.as_str() { + "SET" => { // We don't detect set statements in transactions // No great way to differentiate between set and set local // As a result, we will miss cases when set statements are used in transactions @@ -865,7 +988,8 @@ impl Server { self.cleanup_state.needs_cleanup_set = true; } } - "PREPARE\0" => { + + "PREPARE" => { debug!("Server connection marked for clean up"); self.cleanup_state.needs_cleanup_prepare = true; } @@ -879,6 +1003,17 @@ impl Server { } } + 'S' => { + let key = message.read_string().unwrap(); + let value = message.read_string().unwrap(); + + if let Some(client_server_parameters) = client_server_parameters.as_mut() { + client_server_parameters.set_param(key.clone(), value.clone(), false); + } + + self.server_parameters.set_param(key, value, false); + } + // DataRow 'D' => { // More data is available after this message, this is not the end of the reply. @@ -1089,9 +1224,28 @@ impl Server { } /// Get server startup information to forward it to the client. - /// Not used at the moment. - pub fn server_info(&self) -> BytesMut { - self.server_info.clone() + pub fn server_parameters(&self) -> ServerParameters { + self.server_parameters.clone() + } + + pub async fn sync_parameters(&mut self, parameters: &ServerParameters) -> Result<(), Error> { + let parameter_diff = self.server_parameters.compare_params(parameters); + + if parameter_diff.is_empty() { + return Ok(()); + } + + let mut query = String::from(""); + + for (key, value) in parameter_diff { + query.push_str(&format!("SET {} TO '{}';", key, value)); + } + + let res = self.query(&query).await; + + self.cleanup_state.reset(); + + res } /// Indicate that this server connection cannot be re-used and must be discarded. @@ -1125,7 +1279,7 @@ impl Server { self.send(&query).await?; loop { - let _ = self.recv().await?; + let _ = self.recv(None).await?; if !self.data_available { break; @@ -1166,24 +1320,6 @@ impl Server { Ok(()) } - /// A shorthand for `SET application_name = $1`. - pub async fn set_name(&mut self, name: &str) -> Result<(), Error> { - if self.application_name != name { - self.application_name = name.to_string(); - // We don't want `SET application_name` to mark the server connection - // as needing cleanup - let needs_cleanup_before = self.cleanup_state; - - let result = Ok(self - .query(&format!("SET application_name = '{}'", name)) - .await?); - self.cleanup_state = needs_cleanup_before; - result - } else { - Ok(()) - } - } - /// get Server stats pub fn stats(&self) -> Arc { self.stats.clone() @@ -1241,7 +1377,7 @@ impl Server { .await?; debug!("Connected!, sending query."); server.send(&simple_query(query)).await?; - let mut message = server.recv().await?; + let mut message = server.recv(None).await?; Ok(parse_query_message(&mut message).await?) } diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index e1dbea8bf..dd3fd0529 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -112,10 +112,16 @@ def admin_connection_string "postgresql://#{username}:#{password}@0.0.0.0:#{@port}/pgcat" end - def connection_string(pool_name, username, password = nil) + def connection_string(pool_name, username, password = nil, parameters: {}) cfg = current_config user_idx, user_obj = cfg["pools"][pool_name]["users"].detect { |k, user| user["username"] == username } - "postgresql://#{username}:#{password || user_obj["password"]}@0.0.0.0:#{@port}/#{pool_name}" + connection_string = "postgresql://#{username}:#{password || user_obj["password"]}@0.0.0.0:#{@port}/#{pool_name}" + + # Add the additional parameters to the connection string + parameter_string = parameters.map { |key, value| "#{key}=#{value}" }.join("&") + connection_string += "?#{parameter_string}" unless parameter_string.empty? + + connection_string end def example_connection_string diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index fe216e5bf..628680bd9 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -294,6 +294,30 @@ expect(processes.primary.count_query("DISCARD ALL")).to eq(10) end + + it "Respects tracked parameters on startup" do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user", parameters: { "application_name" => "my_pgcat_test" })) + + expect(conn.async_exec("SHOW application_name")[0]["application_name"]).to eq("my_pgcat_test") + conn.close + end + + it "Respect tracked parameter on set statemet" do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + + conn.async_exec("SET application_name to 'my_pgcat_test'") + expect(conn.async_exec("SHOW application_name")[0]["application_name"]).to eq("my_pgcat_test") + end + + + it "Ignore untracked parameter on set statemet" do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + orignal_statement_timeout = conn.async_exec("SHOW statement_timeout")[0]["statement_timeout"] + + conn.async_exec("SET statement_timeout to 1500") + expect(conn.async_exec("SHOW statement_timeout")[0]["statement_timeout"]).to eq(orignal_statement_timeout) + end + end context "transaction mode with transactions" do From 78ef6ab2cd38cda0bc7ee9741f1709ab1c397928 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 10 Aug 2023 12:25:43 -0700 Subject: [PATCH 451/551] Add pgcat user (#546) * Add pgcat user * warn * dev --- .github/workflows/publish-deb-package.yml | 2 +- Cargo.lock | 2 +- Cargo.toml | 2 +- postinst | 5 +++++ postrm | 1 + prerm | 1 + src/server.rs | 2 +- 7 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-deb-package.yml b/.github/workflows/publish-deb-package.yml index b46b37daa..bfd29f280 100644 --- a/.github/workflows/publish-deb-package.yml +++ b/.github/workflows/publish-deb-package.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "1.1.1" + default: "1.1.2-dev" jobs: build: strategy: diff --git a/Cargo.lock b/Cargo.lock index ec8493293..459312c2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -990,7 +990,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pgcat" -version = "1.1.1" +version = "1.1.2-dev" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index c3db48aa7..805a4c7a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.1.1" +version = "1.1.2-dev" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/postinst b/postinst index 54c04119b..50138c2b2 100644 --- a/postinst +++ b/postinst @@ -1,4 +1,9 @@ #!/bin/bash +set -e systemctl daemon-reload systemctl enable pgcat + +if ! id pgcat 2> /dev/null; then + useradd -s /usr/bin/false pgcat +fi diff --git a/postrm b/postrm index ab20f7b0f..ec1d0f7e1 100644 --- a/postrm +++ b/postrm @@ -1,3 +1,4 @@ #!/bin/bash +set -e systemctl daemon-reload diff --git a/prerm b/prerm index 6dd0e3ac4..601b69914 100644 --- a/prerm +++ b/prerm @@ -1,4 +1,5 @@ #!/bin/bash +set -e systemctl stop pgcat systemctl disable pgcat diff --git a/src/server.rs b/src/server.rs index c4d7a1af4..b006e77b6 100644 --- a/src/server.rs +++ b/src/server.rs @@ -9,7 +9,7 @@ use postgres_protocol::message; use std::collections::{BTreeSet, HashMap, HashSet}; use std::mem; use std::net::IpAddr; -use std::sync::{Arc, Once}; +use std::sync::Arc; use std::time::SystemTime; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, BufStream}; use tokio::net::TcpStream; From ebaf1fd5de43aff1c6c821e862fbee6b2a6ce8a7 Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:08:48 -0400 Subject: [PATCH 452/551] Reset instead of discard all (#549) * Use reset all instead of discard all * Move 'X' handling to before admin handle * fix tests --- src/client.rs | 16 ++++++++-------- src/server.rs | 25 +++++++++++++++++-------- tests/ruby/misc_spec.rb | 20 ++++++++++---------- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/client.rs b/src/client.rs index 6cdea9878..3f6ba13ff 100644 --- a/src/client.rs +++ b/src/client.rs @@ -821,6 +821,14 @@ where message_result = read_message(&mut self.read) => message_result? }; + if message[0] as char == 'X' { + debug!("Client disconnecting"); + + self.stats.disconnect(); + + return Ok(()); + } + // Handle admin database queries. if self.admin { debug!("Handling admin command"); @@ -940,14 +948,6 @@ where continue; } - 'X' => { - debug!("Client disconnecting"); - - self.stats.disconnect(); - - return Ok(()); - } - // Close (F) 'C' => { if prepared_statements_enabled { diff --git a/src/server.rs b/src/server.rs index b006e77b6..dff923191 100644 --- a/src/server.rs +++ b/src/server.rs @@ -107,10 +107,10 @@ impl StreamInner { #[derive(Copy, Clone)] struct CleanupState { - /// If server connection requires DISCARD ALL before checkin because of set statement + /// If server connection requires RESET ALL before checkin because of set statement needs_cleanup_set: bool, - /// If server connection requires DISCARD ALL before checkin because of prepare statement + /// If server connection requires DEALLOCATE ALL before checkin because of prepare statement needs_cleanup_prepare: bool, } @@ -296,7 +296,7 @@ pub struct Server { /// Is the server broken? We'll remote it from the pool if so. bad: bool, - /// If server connection requires DISCARD ALL before checkin + /// If server connection requires reset statements before checkin cleanup_state: CleanupState, /// Mapping of clients and servers used for query cancellation. @@ -982,7 +982,7 @@ impl Server { // We don't detect set statements in transactions // No great way to differentiate between set and set local // As a result, we will miss cases when set statements are used in transactions - // This will reduce amount of discard statements sent + // This will reduce amount of reset statements sent if !self.in_transaction { debug!("Server connection marked for clean up"); self.cleanup_state.needs_cleanup_set = true; @@ -1304,12 +1304,21 @@ impl Server { // Client disconnected but it performed session-altering operations such as // SET statement_timeout to 1 or create a prepared statement. We clear that // to avoid leaking state between clients. For performance reasons we only - // send `DISCARD ALL` if we think the session is altered instead of just sending + // send `RESET ALL` if we think the session is altered instead of just sending // it before each checkin. if self.cleanup_state.needs_cleanup() && self.cleanup_connections { info!(target: "pgcat::server::cleanup", "Server returned with session state altered, discarding state ({}) for application {}", self.cleanup_state, self.application_name); - self.query("DISCARD ALL").await?; - self.query("RESET ROLE").await?; + let mut reset_string = String::from("RESET ROLE;"); + + if self.cleanup_state.needs_cleanup_set { + reset_string.push_str("RESET ALL;"); + }; + + if self.cleanup_state.needs_cleanup_prepare { + reset_string.push_str("DEALLOCATE ALL;"); + }; + + self.query(&reset_string).await?; self.cleanup_state.reset(); } @@ -1336,7 +1345,7 @@ impl Server { self.last_activity } - // Marks a connection as needing DISCARD ALL at checkin + // Marks a connection as needing cleanup at checkin pub fn mark_dirty(&mut self) { self.cleanup_state.set_true(); } diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index 628680bd9..1d4ade4c9 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -221,7 +221,7 @@ conn.close end - it "Does not send DISCARD ALL unless necessary" do + it "Does not send RESET ALL unless necessary" do 10.times do conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) conn.async_exec("SET SERVER ROLE to 'primary'") @@ -229,7 +229,7 @@ conn.close end - expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + expect(processes.primary.count_query("RESET ALL")).to eq(0) 10.times do conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) @@ -239,7 +239,7 @@ conn.close end - expect(processes.primary.count_query("DISCARD ALL")).to eq(10) + expect(processes.primary.count_query("RESET ALL")).to eq(10) end it "Resets server roles correctly" do @@ -273,7 +273,7 @@ end end - it "Does not send DISCARD ALL unless necessary" do + it "Does not send RESET ALL unless necessary" do 10.times do conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) conn.async_exec("SET SERVER ROLE to 'primary'") @@ -282,7 +282,7 @@ conn.close end - expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + expect(processes.primary.count_query("RESET ALL")).to eq(0) 10.times do conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) @@ -292,7 +292,7 @@ conn.close end - expect(processes.primary.count_query("DISCARD ALL")).to eq(10) + expect(processes.primary.count_query("RESET ALL")).to eq(10) end it "Respects tracked parameters on startup" do @@ -331,7 +331,7 @@ conn.async_exec("COMMIT") conn.close end - expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + expect(processes.primary.count_query("RESET ALL")).to eq(0) 10.times do conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) @@ -341,7 +341,7 @@ conn.async_exec("COMMIT") conn.close end - expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + expect(processes.primary.count_query("RESET ALL")).to eq(0) end end @@ -355,7 +355,7 @@ conn.close puts processes.pgcat.logs - expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + expect(processes.primary.count_query("RESET ALL")).to eq(0) end it "will not clean up prepared statements" do @@ -366,7 +366,7 @@ conn.close puts processes.pgcat.logs - expect(processes.primary.count_query("DISCARD ALL")).to eq(0) + expect(processes.primary.count_query("RESET ALL")).to eq(0) end end end From 4b2ed924b677ab677cee81b376bd3a228ebb3f7b Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:01:21 -0400 Subject: [PATCH 453/551] Adds option to log which parameter status is changed by the client (#550) --- src/config.rs | 8 ++++++++ src/mirrors.rs | 1 + src/pool.rs | 7 +++++++ src/server.rs | 9 +++++++++ 4 files changed, 25 insertions(+) diff --git a/src/config.rs b/src/config.rs index 0e4b8c7d8..dc915d570 100644 --- a/src/config.rs +++ b/src/config.rs @@ -547,6 +547,9 @@ pub struct Pool { #[serde(default = "Pool::default_cleanup_server_connections")] pub cleanup_server_connections: bool, + #[serde(default)] // False + pub log_client_parameter_status_changes: bool, + pub plugins: Option, pub shards: BTreeMap, pub users: BTreeMap, @@ -696,6 +699,7 @@ impl Default for Pool { server_lifetime: None, plugins: None, cleanup_server_connections: true, + log_client_parameter_status_changes: false, } } } @@ -1157,6 +1161,10 @@ impl Config { "[pool: {}] Cleanup server connections: {}", pool_name, pool_config.cleanup_server_connections ); + info!( + "[pool: {}] Log client parameter status changes: {}", + pool_name, pool_config.log_client_parameter_status_changes + ); info!( "[pool: {}] Plugins: {}", pool_name, diff --git a/src/mirrors.rs b/src/mirrors.rs index 7922e6f88..90bcd3555 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -41,6 +41,7 @@ impl MirroredClient { Arc::new(RwLock::new(None)), None, true, + false, ); Pool::builder() diff --git a/src/pool.rs b/src/pool.rs index b3627448a..7e110ce2f 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -371,6 +371,7 @@ impl ConnectionPool { None => config.plugins.clone(), }, pool_config.cleanup_server_connections, + pool_config.log_client_parameter_status_changes, ); let connect_timeout = match pool_config.connect_timeout { @@ -956,6 +957,9 @@ pub struct ServerPool { /// Should we clean up dirty connections before putting them into the pool? cleanup_connections: bool, + + /// Log client parameter status changes + log_client_parameter_status_changes: bool, } impl ServerPool { @@ -967,6 +971,7 @@ impl ServerPool { auth_hash: Arc>>, plugins: Option, cleanup_connections: bool, + log_client_parameter_status_changes: bool, ) -> ServerPool { ServerPool { address, @@ -976,6 +981,7 @@ impl ServerPool { auth_hash, plugins, cleanup_connections, + log_client_parameter_status_changes, } } } @@ -1005,6 +1011,7 @@ impl ManageConnection for ServerPool { stats.clone(), self.auth_hash.clone(), self.cleanup_connections, + self.log_client_parameter_status_changes, ) .await { diff --git a/src/server.rs b/src/server.rs index dff923191..70c8270d6 100644 --- a/src/server.rs +++ b/src/server.rs @@ -322,6 +322,9 @@ pub struct Server { /// Should clean up dirty connections? cleanup_connections: bool, + /// Log client parameter status changes + log_client_parameter_status_changes: bool, + /// Prepared statements prepared_statements: BTreeSet, } @@ -337,6 +340,7 @@ impl Server { stats: Arc, auth_hash: Arc>>, cleanup_connections: bool, + log_client_parameter_status_changes: bool, ) -> Result { let cached_resolver = CACHED_RESOLVER.load(); let mut addr_set: Option = None; @@ -825,6 +829,7 @@ impl Server { )), }, cleanup_connections, + log_client_parameter_status_changes, prepared_statements: BTreeSet::new(), }; @@ -1009,6 +1014,9 @@ impl Server { if let Some(client_server_parameters) = client_server_parameters.as_mut() { client_server_parameters.set_param(key.clone(), value.clone(), false); + if self.log_client_parameter_status_changes { + info!("Client parameter status change: {} = {}", key, value) + } } self.server_parameters.set_param(key, value, false); @@ -1382,6 +1390,7 @@ impl Server { Arc::new(ServerStats::default()), Arc::new(RwLock::new(None)), true, + false, ) .await?; debug!("Connected!, sending query."); From bd91d233aff2a01509bf668228e2193c41dbd31f Mon Sep 17 00:00:00 2001 From: Cluas Date: Mon, 21 Aug 2023 10:14:19 +0800 Subject: [PATCH 454/551] docs: complete the missing general items (#553) docs: complete the missing general items. --- CONFIG.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/CONFIG.md b/CONFIG.md index c60e15625..fc118cb47 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -57,6 +57,38 @@ default: 86400000 # 24 hours Max connection lifetime before it's closed, even if actively used. +### server_round_robin +``` +path: general.server_round_robin +default: false +``` + +Whether to use round robin for server selection or not. + +### server_tls +``` +path: general.server_tls +default: false +``` + +Whether to use TLS for server connections or not. + +### verify_server_certificate +``` +path: general.verify_server_certificate +default: false +``` + +Whether to verify server certificate or not. + +### verify_config +``` +path: general.verify_config +default: true +``` + +Whether to verify config or not. + ### idle_client_in_transaction_timeout ``` path: general.idle_client_in_transaction_timeout @@ -194,6 +226,55 @@ default: "admin_pass" Password to access the virtual administrative database +### auth_query +``` +path: general.auth_query +default: +example: "SELECT $1" +``` + +Query to be sent to servers to obtain the hash used for md5 authentication. The connection will be +established using the database configured in the pool. This parameter is inherited by every pool +and can be redefined in pool configuration. + +### auth_query_user +``` +path: general.auth_query_user +default: +example: "sharding_user" +``` + +User to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +specified in `auth_query_user`. The connection will be established using the database configured in the pool. +This parameter is inherited by every pool and can be redefined in pool configuration. + +### auth_query_password +``` +path: general.auth_query_password +default: +example: "sharding_user" +``` + +Password to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query +specified in `auth_query_user`. The connection will be established using the database configured in the pool. +This parameter is inherited by every pool and can be redefined in pool configuration. + +### prepared_statements +``` +path: general.prepared_statements +default: false +``` + +Whether to use prepared statements or not. + +### prepared_statements_cache_size +``` +path: general.prepared_statements_cache_size +default: 500 +``` + +Size of the prepared statements cache. + ### dns_cache_enabled ``` path: general.dns_cache_enabled From 77f9780ef6818f90dfb0df5b95a3c987e360c25d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:41:09 -0700 Subject: [PATCH 455/551] chore(deps): bump rustls-webpki from 0.100.1 to 0.100.2 (#555) Bumps [rustls-webpki](https://github.com/rustls/webpki) from 0.100.1 to 0.100.2. - [Release notes](https://github.com/rustls/webpki/releases) - [Commits](https://github.com/rustls/webpki/compare/v/0.100.1...v/0.100.2) --- updated-dependencies: - dependency-name: rustls-webpki dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 459312c2a..929f1a81e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1310,9 +1310,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.100.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" dependencies = [ "ring", "untrusted", @@ -2002,7 +2002,7 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ - "rustls-webpki 0.100.1", + "rustls-webpki 0.100.2", ] [[package]] From 9843816a860148d8c07e3aff05e8305e56ce883c Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:25:43 -0400 Subject: [PATCH 456/551] Fixes try_execute_command message parsing bug (#560) * Fixes try_execute_command message parsing bug * Fix initial segment logic * Add test --- src/query_router.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/query_router.rs b/src/query_router.rs index 9676a26f6..efca499fd 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -19,9 +19,9 @@ use crate::plugins::{Intercept, Plugin, PluginOutput, QueryLogger, TableAccess}; use crate::pool::PoolSettings; use crate::sharding::Sharder; -use std::cmp; use std::collections::BTreeSet; use std::io::Cursor; +use std::{cmp, mem}; /// Regexes used to parse custom commands. const CUSTOM_SQL_REGEXES: [&str; 7] = [ @@ -141,6 +141,7 @@ impl QueryRouter { let mut message_cursor = Cursor::new(message_buffer); let code = message_cursor.get_u8() as char; + let len = message_cursor.get_i32() as usize; // Check for any sharding regex matches in any queries match code as char { @@ -150,9 +151,13 @@ impl QueryRouter { || self.pool_settings.sharding_key_regex.is_some() { // Check only the first block of bytes configured by the pool settings - let len = message_cursor.get_i32() as usize; let seg = cmp::min(len - 5, self.pool_settings.regex_search_limit); - let initial_segment = String::from_utf8_lossy(&message_buffer[0..seg]); + + let query_start_index = mem::size_of::() + mem::size_of::(); + + let initial_segment = String::from_utf8_lossy( + &message_buffer[query_start_index..query_start_index + seg], + ); // Check for a shard_id included in the query if let Some(shard_id_regex) = &self.pool_settings.shard_id_regex { @@ -192,7 +197,6 @@ impl QueryRouter { return None; } - let _len = message_cursor.get_i32() as usize; let query = message_cursor.read_string().unwrap(); let regex_set = match CUSTOM_SQL_REGEX_SET.get() { @@ -1291,6 +1295,11 @@ mod test { // Shard should start out unset assert_eq!(qr.active_shard, None); + // Don't panic when short query eg. ; is sent + let q0 = simple_query(";"); + assert!(qr.try_execute_command(&q0) == None); + assert_eq!(qr.active_shard, None); + // Make sure setting it works let q1 = simple_query("/* shard_id: 1 */ select 1 from foo;"); assert!(qr.try_execute_command(&q1) == None); From 49e3cb06cc88e2adcc56c55ca5df2b9921d4280b Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Fri, 25 Aug 2023 13:33:39 -0400 Subject: [PATCH 457/551] Don't unwrap stream.peer_addr() (#562) --- src/client.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index 3f6ba13ff..8ad0d47b1 100644 --- a/src/client.rs +++ b/src/client.rs @@ -348,7 +348,15 @@ pub async fn startup_tls( ) -> Result>, WriteHalf>>, Error> { // Negotiate TLS. let tls = Tls::new()?; - let addr = stream.peer_addr().unwrap(); + let addr = match stream.peer_addr() { + Ok(addr) => addr, + Err(err) => { + return Err(Error::SocketError(format!( + "Failed to get peer address: {:?}", + err + ))); + } + }; let mut stream = match tls.acceptor.accept(stream).await { Ok(stream) => stream, From f5ff6236ed40afafe17bb56cc850af794bdcb51c Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 28 Aug 2023 21:20:53 -0500 Subject: [PATCH 458/551] Add yj to CI image (#568) --- Dockerfile.ci | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.ci b/Dockerfile.ci index 4503e8703..a750612c7 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,6 @@ FROM cimg/rust:1.67.1 +COPY --from=sclevine/yj /bin/yj /bin/yj +RUN /bin/yj -h RUN sudo apt-get update && \ sudo apt-get install -y \ psmisc postgresql-contrib-14 postgresql-client-14 libpq-dev \ From 843d09b82249bb329f34413b9ac8317119dcce3b Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 29 Aug 2023 10:07:36 -0700 Subject: [PATCH 459/551] Allow pause/resuming all pools (#566) support pausing all pools --- src/admin.rs | 168 +++++++++++++++++++++++---------------- tests/ruby/admin_spec.rb | 24 ++++++ 2 files changed, 124 insertions(+), 68 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index f27b2a0d1..da9252920 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -74,11 +74,11 @@ where } "PAUSE" => { trace!("PAUSE"); - pause(stream, query_parts[1]).await + pause(stream, query_parts).await } "RESUME" => { trace!("RESUME"); - resume(stream, query_parts[1]).await + resume(stream, query_parts).await } "SHUTDOWN" => { trace!("SHUTDOWN"); @@ -797,96 +797,128 @@ where } /// Pause a pool. It won't pass any more queries to the backends. -async fn pause(stream: &mut T, query: &str) -> Result<(), Error> +async fn pause(stream: &mut T, tokens: Vec<&str>) -> Result<(), Error> where T: tokio::io::AsyncWrite + std::marker::Unpin, { - let parts: Vec<&str> = query.split(",").map(|part| part.trim()).collect(); - - if parts.len() != 2 { - error_response( - stream, - "PAUSE requires a database and a user, e.g. PAUSE my_db,my_user", - ) - .await - } else { - let database = parts[0]; - let user = parts[1]; - - match get_pool(database, user) { - Some(pool) => { - pool.pause(); + let parts: Vec<&str> = match tokens.len() == 2 { + true => tokens[1].split(",").map(|part| part.trim()).collect(), + false => Vec::new(), + }; - let mut res = BytesMut::new(); + match parts.len() { + 0 => { + for (_, pool) in get_all_pools() { + pool.pause(); + } - res.put(command_complete(&format!("PAUSE {},{}", database, user))); + let mut res = BytesMut::new(); - // ReadyForQuery - res.put_u8(b'Z'); - res.put_i32(5); - res.put_u8(b'I'); + res.put(command_complete("PAUSE")); - write_all_half(stream, &res).await - } + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); - None => { - error_response( - stream, - &format!( - "No pool configured for database: {}, user: {}", - database, user - ), - ) - .await + write_all_half(stream, &res).await + } + 2 => { + let database = parts[0]; + let user = parts[1]; + + match get_pool(database, user) { + Some(pool) => { + pool.pause(); + + let mut res = BytesMut::new(); + + res.put(command_complete(&format!("PAUSE {},{}", database, user))); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await + } + + None => { + error_response( + stream, + &format!( + "No pool configured for database: {}, user: {}", + database, user + ), + ) + .await + } } } + _ => error_response(stream, "usage: PAUSE [db, user]").await, } } /// Resume a pool. Queries are allowed again. -async fn resume(stream: &mut T, query: &str) -> Result<(), Error> +async fn resume(stream: &mut T, tokens: Vec<&str>) -> Result<(), Error> where T: tokio::io::AsyncWrite + std::marker::Unpin, { - let parts: Vec<&str> = query.split(",").map(|part| part.trim()).collect(); - - if parts.len() != 2 { - error_response( - stream, - "RESUME requires a database and a user, e.g. RESUME my_db,my_user", - ) - .await - } else { - let database = parts[0]; - let user = parts[1]; - - match get_pool(database, user) { - Some(pool) => { - pool.resume(); + let parts: Vec<&str> = match tokens.len() == 2 { + true => tokens[1].split(",").map(|part| part.trim()).collect(), + false => Vec::new(), + }; - let mut res = BytesMut::new(); + match parts.len() { + 0 => { + for (_, pool) in get_all_pools() { + pool.resume(); + } - res.put(command_complete(&format!("RESUME {},{}", database, user))); + let mut res = BytesMut::new(); - // ReadyForQuery - res.put_u8(b'Z'); - res.put_i32(5); - res.put_u8(b'I'); + res.put(command_complete("RESUME")); - write_all_half(stream, &res).await - } + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); - None => { - error_response( - stream, - &format!( - "No pool configured for database: {}, user: {}", - database, user - ), - ) - .await + write_all_half(stream, &res).await + } + 2 => { + let database = parts[0]; + let user = parts[1]; + + match get_pool(database, user) { + Some(pool) => { + pool.resume(); + + let mut res = BytesMut::new(); + + res.put(command_complete(&format!("RESUME {},{}", database, user))); + + // ReadyForQuery + res.put_u8(b'Z'); + res.put_i32(5); + res.put_u8(b'I'); + + write_all_half(stream, &res).await + } + + None => { + error_response( + stream, + &format!( + "No pool configured for database: {}, user: {}", + database, user + ), + ) + .await + } } } + _ => error_response(stream, "usage: RESUME [db, user]").await, } } diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index 9a8523500..abaa5ff4d 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -90,4 +90,28 @@ expect(results["pool_mode"]).to eq("transaction") end end + + describe "PAUSE" do + it "pauses all pools" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW DATABASES").to_a + expect(results.map{ |r| r["paused"] }.uniq).to eq(["0"]) + + admin_conn.async_exec("PAUSE") + + results = admin_conn.async_exec("SHOW DATABASES").to_a + expect(results.map{ |r| r["paused"] }.uniq).to eq(["1"]) + + admin_conn.async_exec("RESUME") + + results = admin_conn.async_exec("SHOW DATABASES").to_a + expect(results.map{ |r| r["paused"] }.uniq).to eq(["0"]) + end + + it "handles errors" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + expect { admin_conn.async_exec("PAUSE foo").to_a }.to raise_error(PG::SystemError) + expect { admin_conn.async_exec("PAUSE foo,bar").to_a }.to raise_error(PG::SystemError) + end + end end From 18455ebb394ada72ccca39417c89a9b93450a923 Mon Sep 17 00:00:00 2001 From: hi019 <65871571+hi019@users.noreply.github.com> Date: Wed, 30 Aug 2023 19:51:31 -0400 Subject: [PATCH 460/551] Fix Docker image runs erroring due to glibc incompatability (#572) Fix Docker image builds breaking due to glibc incompatability --- Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ec29182a8..f2d580629 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,13 @@ -FROM rust:1 AS builder +FROM rust:1-slim-bookworm AS builder + +RUN apt-get update && \ + apt-get install -y build-essential + COPY . /app WORKDIR /app RUN cargo build --release -FROM debian:bullseye-slim +FROM debian:bookworm-slim COPY --from=builder /app/target/release/pgcat /usr/bin/pgcat COPY --from=builder /app/pgcat.toml /etc/pgcat/pgcat.toml WORKDIR /etc/pgcat From 9f4cac9524b651722c3564aecc09fea591b6acff Mon Sep 17 00:00:00 2001 From: hellower Date: Fri, 1 Sep 2023 06:11:38 +0900 Subject: [PATCH 461/551] stream.peer_addr() & auth_query (#575) * Don't unwrap stream.peer_addr() https://github.com/postgresml/pgcat/pull/562 (same code) (another lines changed) * auth_query (real sample) # single quote need auth_query="SELECT usename, passwd FROM pg_shadow WHERE usename='$1'" --- pgcat.toml | 2 +- src/client.rs | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pgcat.toml b/pgcat.toml index ae5d74dce..654f5e89a 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -187,7 +187,7 @@ sharding_function = "pg_bigint_hash" # Query to be sent to servers to obtain the hash used for md5 authentication. The connection will be # established using the database configured in the pool. This parameter is inherited by every pool # and can be redefined in pool configuration. -# auth_query = "SELECT $1" +# auth_query="SELECT usename, passwd FROM pg_shadow WHERE usename='$1'" # User to be used for connecting to servers to obtain the hash used for md5 authentication by sending the query # specified in `auth_query_user`. The connection will be established using the database configured in the pool. diff --git a/src/client.rs b/src/client.rs index 8ad0d47b1..8edecea1d 100644 --- a/src/client.rs +++ b/src/client.rs @@ -117,7 +117,15 @@ pub async fn client_entrypoint( log_client_connections: bool, ) -> Result<(), Error> { // Figure out if the client wants TLS or not. - let addr = stream.peer_addr().unwrap(); + let addr = match stream.peer_addr() { + Ok(addr) => addr, + Err(err) => { + return Err(Error::SocketError(format!( + "Failed to get peer address: {:?}", + err + ))); + } + }; match get_startup::(&mut stream).await { // Client requested a TLS connection. From 7c2ccc1d64b1f38065cb041424eb5cf425702338 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Mon, 11 Sep 2023 13:47:28 -0500 Subject: [PATCH 462/551] Allow configuring routing decision when no shard is selected (#578) The TL;DR for the change is that we allow QueryRouter to set the active shard to None. This signals to the Pool::get method that we have no shard selected. The get method follows a no_shard_specified_behavior config to know how to route the query. Original PR description Ruby-pg library makes a startup query to SET client_encoding to ... if Encoding.default_internal value is set (Code). This query is troublesome because we cannot possibly attach a routing comment to it. PgCat, by default, will route that query to the default shard. Everything is fine until shard 0 has issues, Clients will all be attempting to send this query to shard0 which increases the connection latency significantly for all clients, even those not interested in shard0 This PR introduces no_shard_specified_behavior that defines the behavior in case we have routing-by-comment enabled but we get a query without a comment. The allowed behaviors are random: Picks a shard at random random_healthy: Picks a shard at random favoring shards with the least number of recent connection/checkout errors shard_: e.g. shard_0, shard_4, etc. picks a specific shard, everytime In order to achieve this, this PR introduces an error_count on the Address Object that tracks the number of errors since the last checkout and uses that metric to sort shards by error count before making a routing decision. I didn't want to use address stats to avoid introducing a routing dependency on internal stats (We might do that in the future but I prefer to avoid this for the time being. I also made changes to the test environment to replace Ruby's TOML reader library, It appears to be abandoned and does not support mixed arrays (which we use in the config toml), and it also does not play nicely with single-quoted regular expressions. I opted for using yj which is a CLI tool that can convert from toml to JSON and back. So I refactor the tests to use that library. --- .circleci/config.yml | 2 +- pgcat.toml | 6 + src/client.rs | 47 ++++---- src/config.rs | 81 +++++++++++++ src/errors.rs | 1 + src/pool.rs | 79 +++++++++++-- src/query_router.rs | 57 +++++---- tests/docker/Dockerfile | 2 + tests/ruby/auth_query_spec.rb | 2 +- tests/ruby/helpers/auth_query_helper.rb | 16 +-- tests/ruby/helpers/pg_instance.rb | 24 +++- tests/ruby/helpers/pgcat_helper.rb | 16 +-- tests/ruby/helpers/pgcat_process.rb | 44 ++++--- tests/ruby/mirrors_spec.rb | 15 +-- tests/ruby/misc_spec.rb | 17 ++- tests/ruby/sharding_spec.rb | 151 +++++++++++++++++++++++- 16 files changed, 448 insertions(+), 112 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c7f5c9fa8..c8344911f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - - image: ghcr.io/levkk/pgcat-ci:1.67 + - image: ghcr.io/postgresml/pgcat-ci:latest environment: RUST_LOG: info LLVM_PROFILE_FILE: /tmp/pgcat-%m-%p.profraw diff --git a/pgcat.toml b/pgcat.toml index 654f5e89a..772a1365e 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -177,6 +177,12 @@ primary_reads_enabled = true # shard_id_regex = '/\* shard_id: (\d+) \*/' # regex_search_limit = 1000 # only look at the first 1000 characters of SQL statements +# Defines the behavior when no shard is selected in a sharded system. +# `random`: picks a shard at random +# `random_healthy`: picks a shard at random favoring shards with the least number of recent errors +# `shard_`: e.g. shard_0, shard_4, etc. picks a specific shard, everytime +# no_shard_specified_behavior = "shard_0" + # So what if you wanted to implement a different hashing function, # or you've already built one and you want this pooler to use it? # Current options: diff --git a/src/client.rs b/src/client.rs index 8edecea1d..4b2811216 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1009,23 +1009,27 @@ where // SET SHARD TO Some((Command::SetShard, _)) => { - // Selected shard is not configured. - if query_router.shard() >= pool.shards() { - // Set the shard back to what it was. - query_router.set_shard(current_shard); - - error_response( - &mut self.write, - &format!( - "shard {} is more than configured {}, staying on shard {} (shard numbers start at 0)", - query_router.shard(), - pool.shards(), - current_shard, - ), - ) - .await?; - } else { - custom_protocol_response_ok(&mut self.write, "SET SHARD").await?; + match query_router.shard() { + None => (), + Some(selected_shard) => { + if selected_shard >= pool.shards() { + // Bad shard number, send error message to client. + query_router.set_shard(current_shard); + + error_response( + &mut self.write, + &format!( + "shard {} is not configured {}, staying on shard {:?} (shard numbers start at 0)", + selected_shard, + pool.shards(), + current_shard, + ), + ) + .await?; + } else { + custom_protocol_response_ok(&mut self.write, "SET SHARD").await?; + } + } } continue; } @@ -1093,8 +1097,11 @@ where self.buffer.clear(); } - error_response(&mut self.write, "could not get connection from the pool") - .await?; + error_response( + &mut self.write, + format!("could not get connection from the pool - {}", err).as_str(), + ) + .await?; error!( "Could not get connection from pool: \ @@ -1234,7 +1241,7 @@ where {{ \ pool_name: {}, \ username: {}, \ - shard: {}, \ + shard: {:?}, \ role: \"{:?}\" \ }}", self.pool_name, diff --git a/src/config.rs b/src/config.rs index dc915d570..0404abc9d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,11 +3,14 @@ use arc_swap::ArcSwap; use log::{error, info}; use once_cell::sync::Lazy; use regex::Regex; +use serde::{Deserializer, Serializer}; use serde_derive::{Deserialize, Serialize}; + use std::collections::hash_map::DefaultHasher; use std::collections::{BTreeMap, HashMap, HashSet}; use std::hash::{Hash, Hasher}; use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use tokio::fs::File; use tokio::io::AsyncReadExt; @@ -101,6 +104,9 @@ pub struct Address { /// Address stats pub stats: Arc, + + /// Number of errors encountered since last successful checkout + pub error_count: Arc, } impl Default for Address { @@ -118,6 +124,7 @@ impl Default for Address { pool_name: String::from("pool_name"), mirrors: Vec::new(), stats: Arc::new(AddressStats::default()), + error_count: Arc::new(AtomicU64::new(0)), } } } @@ -182,6 +189,18 @@ impl Address { ), } } + + pub fn error_count(&self) -> u64 { + self.error_count.load(Ordering::Relaxed) + } + + pub fn increment_error_count(&self) { + self.error_count.fetch_add(1, Ordering::Relaxed); + } + + pub fn reset_error_count(&self) { + self.error_count.store(0, Ordering::Relaxed); + } } /// PostgreSQL user. @@ -540,6 +559,9 @@ pub struct Pool { pub shard_id_regex: Option, pub regex_search_limit: Option, + #[serde(default = "Pool::default_default_shard")] + pub default_shard: DefaultShard, + pub auth_query: Option, pub auth_query_user: Option, pub auth_query_password: Option, @@ -575,6 +597,10 @@ impl Pool { PoolMode::Transaction } + pub fn default_default_shard() -> DefaultShard { + DefaultShard::default() + } + pub fn default_load_balancing_mode() -> LoadBalancingMode { LoadBalancingMode::Random } @@ -666,6 +692,16 @@ impl Pool { None => None, }; + match self.default_shard { + DefaultShard::Shard(shard_number) => { + if shard_number >= self.shards.len() { + error!("Invalid shard {:?}", shard_number); + return Err(Error::BadConfig); + } + } + _ => (), + } + for (_, user) in &self.users { user.validate()?; } @@ -693,6 +729,7 @@ impl Default for Pool { sharding_key_regex: None, shard_id_regex: None, regex_search_limit: Some(1000), + default_shard: Self::default_default_shard(), auth_query: None, auth_query_user: None, auth_query_password: None, @@ -711,6 +748,50 @@ pub struct ServerConfig { pub role: Role, } +// No Shard Specified handling. +#[derive(Debug, PartialEq, Clone, Eq, Hash, Copy)] +pub enum DefaultShard { + Shard(usize), + Random, + RandomHealthy, +} +impl Default for DefaultShard { + fn default() -> Self { + DefaultShard::Shard(0) + } +} +impl serde::Serialize for DefaultShard { + fn serialize(&self, serializer: S) -> Result { + match self { + DefaultShard::Shard(shard) => { + serializer.serialize_str(&format!("shard_{}", &shard.to_string())) + } + DefaultShard::Random => serializer.serialize_str("random"), + DefaultShard::RandomHealthy => serializer.serialize_str("random_healthy"), + } + } +} +impl<'de> serde::Deserialize<'de> for DefaultShard { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + if s.starts_with("shard_") { + let shard = s[6..].parse::().map_err(serde::de::Error::custom)?; + return Ok(DefaultShard::Shard(shard)); + } + + match s.as_str() { + "random" => Ok(DefaultShard::Random), + "random_healthy" => Ok(DefaultShard::RandomHealthy), + _ => Err(serde::de::Error::custom( + "invalid value for no_shard_specified_behavior", + )), + } + } +} + #[derive(Clone, PartialEq, Serialize, Deserialize, Debug, Hash, Eq)] pub struct MirrorServerConfig { pub host: String, diff --git a/src/errors.rs b/src/errors.rs index 014a1340d..a6aebc509 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -28,6 +28,7 @@ pub enum Error { UnsupportedStatement, QueryRouterParserError(String), QueryRouterError(String), + InvalidShardId(usize), } #[derive(Clone, PartialEq, Debug)] diff --git a/src/pool.rs b/src/pool.rs index 7e110ce2f..181234071 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -10,6 +10,7 @@ use rand::thread_rng; use regex::Regex; use std::collections::HashMap; use std::fmt::{Display, Formatter}; +use std::sync::atomic::AtomicU64; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -18,7 +19,7 @@ use std::time::Instant; use tokio::sync::Notify; use crate::config::{ - get_config, Address, General, LoadBalancingMode, Plugins, PoolMode, Role, User, + get_config, Address, DefaultShard, General, LoadBalancingMode, Plugins, PoolMode, Role, User, }; use crate::errors::Error; @@ -140,6 +141,9 @@ pub struct PoolSettings { // Regex for searching for the shard id in SQL statements pub shard_id_regex: Option, + // What to do when no shard is selected in a sharded system + pub default_shard: DefaultShard, + // Limit how much of each query is searched for a potential shard regex match pub regex_search_limit: usize, @@ -173,6 +177,7 @@ impl Default for PoolSettings { sharding_key_regex: None, shard_id_regex: None, regex_search_limit: 1000, + default_shard: DefaultShard::Shard(0), auth_query: None, auth_query_user: None, auth_query_password: None, @@ -299,6 +304,7 @@ impl ConnectionPool { pool_name: pool_name.clone(), mirrors: vec![], stats: Arc::new(AddressStats::default()), + error_count: Arc::new(AtomicU64::new(0)), }); address_id += 1; } @@ -317,6 +323,7 @@ impl ConnectionPool { pool_name: pool_name.clone(), mirrors: mirror_addresses, stats: Arc::new(AddressStats::default()), + error_count: Arc::new(AtomicU64::new(0)), }; address_id += 1; @@ -482,6 +489,7 @@ impl ConnectionPool { .clone() .map(|regex| Regex::new(regex.as_str()).unwrap()), regex_search_limit: pool_config.regex_search_limit.unwrap_or(1000), + default_shard: pool_config.default_shard.clone(), auth_query: pool_config.auth_query.clone(), auth_query_user: pool_config.auth_query_user.clone(), auth_query_password: pool_config.auth_query_password.clone(), @@ -603,19 +611,51 @@ impl ConnectionPool { /// Get a connection from the pool. pub async fn get( &self, - shard: usize, // shard number + shard: Option, // shard number role: Option, // primary or replica client_stats: &ClientStats, // client id ) -> Result<(PooledConnection<'_, ServerPool>, Address), Error> { - let mut candidates: Vec<&Address> = self.addresses[shard] + let effective_shard_id = if self.shards() == 1 { + // The base, unsharded case + Some(0) + } else { + if !self.valid_shard_id(shard) { + // None is valid shard ID so it is safe to unwrap here + return Err(Error::InvalidShardId(shard.unwrap())); + } + shard + }; + + let mut candidates = self + .addresses .iter() + .flatten() .filter(|address| address.role == role) - .collect(); + .collect::>(); - // We shuffle even if least_outstanding_queries is used to avoid imbalance - // in cases where all candidates have more or less the same number of outstanding - // queries + // We start with a shuffled list of addresses even if we end up resorting + // this is meant to avoid hitting instance 0 everytime if the sorting metric + // ends up being the same for all instances candidates.shuffle(&mut thread_rng()); + + match effective_shard_id { + Some(shard_id) => candidates.retain(|address| address.shard == shard_id), + None => match self.settings.default_shard { + DefaultShard::Shard(shard_id) => { + candidates.retain(|address| address.shard == shard_id) + } + DefaultShard::Random => (), + DefaultShard::RandomHealthy => { + candidates.sort_by(|a, b| { + b.error_count + .load(Ordering::Relaxed) + .partial_cmp(&a.error_count.load(Ordering::Relaxed)) + .unwrap() + }); + } + }, + }; + if self.settings.load_balancing_mode == LoadBalancingMode::LeastOutstandingConnections { candidates.sort_by(|a, b| { self.busy_connection_count(b) @@ -651,7 +691,10 @@ impl ConnectionPool { .get() .await { - Ok(conn) => conn, + Ok(conn) => { + address.reset_error_count(); + conn + } Err(err) => { error!( "Connection checkout error for instance {:?}, error: {:?}", @@ -766,6 +809,18 @@ impl ConnectionPool { /// traffic for any new transactions. Existing transactions on that replica /// will finish successfully or error out to the clients. pub fn ban(&self, address: &Address, reason: BanReason, client_info: Option<&ClientStats>) { + // Count the number of errors since the last successful checkout + // This is used to determine if the shard is down + match reason { + BanReason::FailedHealthCheck + | BanReason::FailedCheckout + | BanReason::MessageSendFailed + | BanReason::MessageReceiveFailed => { + address.increment_error_count(); + } + _ => (), + }; + // Primary can never be banned if address.role == Role::Primary { return; @@ -920,6 +975,7 @@ impl ConnectionPool { self.original_server_parameters.read().clone() } + /// Get the number of checked out connection for an address fn busy_connection_count(&self, address: &Address) -> u32 { let state = self.pool_state(address.shard, address.address_index); let idle = state.idle_connections; @@ -933,6 +989,13 @@ impl ConnectionPool { debug!("{:?} has {:?} busy connections", address, busy); return busy; } + + fn valid_shard_id(&self, shard: Option) -> bool { + match shard { + None => true, + Some(shard) => shard < self.shards(), + } + } } /// Wrapper for the bb8 connection pool. diff --git a/src/query_router.rs b/src/query_router.rs index efca499fd..9d7a106a4 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -143,13 +143,14 @@ impl QueryRouter { let code = message_cursor.get_u8() as char; let len = message_cursor.get_i32() as usize; + let comment_shard_routing_enabled = self.pool_settings.shard_id_regex.is_some() + || self.pool_settings.sharding_key_regex.is_some(); + // Check for any sharding regex matches in any queries - match code as char { - // For Parse and Query messages peek to see if they specify a shard_id as a comment early in the statement - 'P' | 'Q' => { - if self.pool_settings.shard_id_regex.is_some() - || self.pool_settings.sharding_key_regex.is_some() - { + if comment_shard_routing_enabled { + match code as char { + // For Parse and Query messages peek to see if they specify a shard_id as a comment early in the statement + 'P' | 'Q' => { // Check only the first block of bytes configured by the pool settings let seg = cmp::min(len - 5, self.pool_settings.regex_search_limit); @@ -166,7 +167,7 @@ impl QueryRouter { }); if let Some(shard_id) = shard_id { debug!("Setting shard to {:?}", shard_id); - self.set_shard(shard_id); + self.set_shard(Some(shard_id)); // Skip other command processing since a sharding command was found return None; } @@ -188,8 +189,8 @@ impl QueryRouter { } } } + _ => {} } - _ => {} } // Only simple protocol supported for commands processed below @@ -248,7 +249,9 @@ impl QueryRouter { } } - Command::ShowShard => self.shard().to_string(), + Command::ShowShard => self + .shard() + .map_or_else(|| "unset".to_string(), |x| x.to_string()), Command::ShowServerRole => match self.active_role { Some(Role::Primary) => Role::Primary.to_string(), Some(Role::Replica) => Role::Replica.to_string(), @@ -581,7 +584,7 @@ impl QueryRouter { // TODO: Support multi-shard queries some day. if shards.len() == 1 { debug!("Found one sharding key"); - self.set_shard(*shards.first().unwrap()); + self.set_shard(Some(*shards.first().unwrap())); true } else { debug!("Found no sharding keys"); @@ -865,7 +868,7 @@ impl QueryRouter { self.pool_settings.sharding_function, ); let shard = sharder.shard(sharding_key); - self.set_shard(shard); + self.set_shard(Some(shard)); self.active_shard } @@ -875,12 +878,12 @@ impl QueryRouter { } /// Get desired shard we should be talking to. - pub fn shard(&self) -> usize { - self.active_shard.unwrap_or(0) + pub fn shard(&self) -> Option { + self.active_shard } - pub fn set_shard(&mut self, shard: usize) { - self.active_shard = Some(shard); + pub fn set_shard(&mut self, shard: Option) { + self.active_shard = shard; } /// Should we attempt to parse queries? @@ -1090,7 +1093,7 @@ mod test { qr.try_execute_command(&query), Some((Command::SetShardingKey, String::from("0"))) ); - assert_eq!(qr.shard(), 0); + assert_eq!(qr.shard().unwrap(), 0); // SetShard let query = simple_query("SET SHARD TO '1'"); @@ -1098,7 +1101,7 @@ mod test { qr.try_execute_command(&query), Some((Command::SetShard, String::from("1"))) ); - assert_eq!(qr.shard(), 1); + assert_eq!(qr.shard().unwrap(), 1); // ShowShard let query = simple_query("SHOW SHARD"); @@ -1204,6 +1207,7 @@ mod test { ban_time: PoolSettings::default().ban_time, sharding_key_regex: None, shard_id_regex: None, + default_shard: crate::config::DefaultShard::Shard(0), regex_search_limit: 1000, auth_query: None, auth_query_password: None, @@ -1281,6 +1285,7 @@ mod test { ban_time: PoolSettings::default().ban_time, sharding_key_regex: Some(Regex::new(r"/\* sharding_key: (\d+) \*/").unwrap()), shard_id_regex: Some(Regex::new(r"/\* shard_id: (\d+) \*/").unwrap()), + default_shard: crate::config::DefaultShard::Shard(0), regex_search_limit: 1000, auth_query: None, auth_query_password: None, @@ -1331,7 +1336,7 @@ mod test { .unwrap(), ) .is_ok()); - assert_eq!(qr.shard(), 2); + assert_eq!(qr.shard().unwrap(), 2); assert!(qr .infer( @@ -1341,7 +1346,7 @@ mod test { .unwrap() ) .is_ok()); - assert_eq!(qr.shard(), 0); + assert_eq!(qr.shard().unwrap(), 0); assert!(qr .infer( @@ -1354,7 +1359,7 @@ mod test { .unwrap() ) .is_ok()); - assert_eq!(qr.shard(), 2); + assert_eq!(qr.shard().unwrap(), 2); // Shard did not move because we couldn't determine the sharding key since it could be ambiguous // in the query. @@ -1366,7 +1371,7 @@ mod test { .unwrap() ) .is_ok()); - assert_eq!(qr.shard(), 2); + assert_eq!(qr.shard().unwrap(), 2); assert!(qr .infer( @@ -1376,7 +1381,7 @@ mod test { .unwrap() ) .is_ok()); - assert_eq!(qr.shard(), 0); + assert_eq!(qr.shard().unwrap(), 0); assert!(qr .infer( @@ -1386,7 +1391,7 @@ mod test { .unwrap() ) .is_ok()); - assert_eq!(qr.shard(), 2); + assert_eq!(qr.shard().unwrap(), 2); // Super unique sharding key qr.pool_settings.automatic_sharding_key = Some("*.unique_enough_column_name".to_string()); @@ -1398,7 +1403,7 @@ mod test { .unwrap() ) .is_ok()); - assert_eq!(qr.shard(), 0); + assert_eq!(qr.shard().unwrap(), 0); assert!(qr .infer( @@ -1406,7 +1411,7 @@ mod test { .unwrap() ) .is_ok()); - assert_eq!(qr.shard(), 0); + assert_eq!(qr.shard().unwrap(), 0); } #[test] @@ -1434,7 +1439,7 @@ mod test { assert_eq!(qr.placeholders.len(), 1); assert!(qr.infer_shard_from_bind(&bind)); - assert_eq!(qr.shard(), 2); + assert_eq!(qr.shard().unwrap(), 2); assert!(qr.placeholders.is_empty()); } diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 99fd694db..261adb05c 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -1,5 +1,7 @@ FROM rust:bullseye +COPY --from=sclevine/yj /bin/yj /bin/yj +RUN /bin/yj -h RUN apt-get update && apt-get install llvm-11 psmisc postgresql-contrib postgresql-client ruby ruby-dev libpq-dev python3 python3-pip lcov curl sudo iproute2 -y RUN cargo install cargo-binutils rustfilt RUN rustup component add llvm-tools-preview diff --git a/tests/ruby/auth_query_spec.rb b/tests/ruby/auth_query_spec.rb index 1ac621648..c1ee744ad 100644 --- a/tests/ruby/auth_query_spec.rb +++ b/tests/ruby/auth_query_spec.rb @@ -185,7 +185,7 @@ }, } } - } + } context 'and with cleartext passwords set' do it 'it uses local passwords' do diff --git a/tests/ruby/helpers/auth_query_helper.rb b/tests/ruby/helpers/auth_query_helper.rb index 60e85713b..43d7c7855 100644 --- a/tests/ruby/helpers/auth_query_helper.rb +++ b/tests/ruby/helpers/auth_query_helper.rb @@ -33,18 +33,18 @@ def self.single_shard_auth_query( "0" => { "database" => "shard0", "servers" => [ - ["localhost", primary.port.to_s, "primary"], - ["localhost", replica.port.to_s, "replica"], + ["localhost", primary.port.to_i, "primary"], + ["localhost", replica.port.to_i, "replica"], ] }, }, "users" => { "0" => user.merge(config_user) } } } - pgcat_cfg["general"]["port"] = pgcat.port + pgcat_cfg["general"]["port"] = pgcat.port.to_i pgcat.update_config(pgcat_cfg) pgcat.start - + pgcat.wait_until_ready( pgcat.connection_string( "sharded_db", @@ -92,13 +92,13 @@ def self.two_pools_auth_query( "0" => { "database" => database, "servers" => [ - ["localhost", primary.port.to_s, "primary"], - ["localhost", replica.port.to_s, "replica"], + ["localhost", primary.port.to_i, "primary"], + ["localhost", replica.port.to_i, "replica"], ] }, }, "users" => { "0" => user.merge(config_user) } - } + } end # Main proxy configs pgcat_cfg["pools"] = { @@ -109,7 +109,7 @@ def self.two_pools_auth_query( pgcat_cfg["general"]["port"] = pgcat.port pgcat.update_config(pgcat_cfg.deep_merge(extra_conf)) pgcat.start - + pgcat.wait_until_ready(pgcat.connection_string("sharded_db0", pg_user['username'], pg_user['password'])) OpenStruct.new.tap do |struct| diff --git a/tests/ruby/helpers/pg_instance.rb b/tests/ruby/helpers/pg_instance.rb index a38282486..53617c240 100644 --- a/tests/ruby/helpers/pg_instance.rb +++ b/tests/ruby/helpers/pg_instance.rb @@ -7,10 +7,24 @@ class PgInstance attr_reader :password attr_reader :database_name + def self.mass_takedown(databases) + raise StandardError "block missing" unless block_given? + + databases.each do |database| + database.toxiproxy.toxic(:limit_data, bytes: 1).toxics.each(&:save) + end + sleep 0.1 + yield + ensure + databases.each do |database| + database.toxiproxy.toxics.each(&:destroy) + end + end + def initialize(port, username, password, database_name) - @original_port = port + @original_port = port.to_i @toxiproxy_port = 10000 + port.to_i - @port = @toxiproxy_port + @port = @toxiproxy_port.to_i @username = username @password = password @@ -48,9 +62,9 @@ def toxiproxy def take_down if block_given? - Toxiproxy[@toxiproxy_name].toxic(:limit_data, bytes: 5).apply { yield } + Toxiproxy[@toxiproxy_name].toxic(:limit_data, bytes: 1).apply { yield } else - Toxiproxy[@toxiproxy_name].toxic(:limit_data, bytes: 5).toxics.each(&:save) + Toxiproxy[@toxiproxy_name].toxic(:limit_data, bytes: 1).toxics.each(&:save) end end @@ -89,6 +103,6 @@ def count_query(query) end def count_select_1_plus_2 - with_connection { |c| c.async_exec("SELECT SUM(calls) FROM pg_stat_statements WHERE query = 'SELECT $1 + $2'")[0]["sum"].to_i } + with_connection { |c| c.async_exec("SELECT SUM(calls) FROM pg_stat_statements WHERE query LIKE '%SELECT $1 + $2%'")[0]["sum"].to_i } end end diff --git a/tests/ruby/helpers/pgcat_helper.rb b/tests/ruby/helpers/pgcat_helper.rb index 9b764d877..9b95dbfa9 100644 --- a/tests/ruby/helpers/pgcat_helper.rb +++ b/tests/ruby/helpers/pgcat_helper.rb @@ -38,9 +38,9 @@ def self.three_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mod "automatic_sharding_key" => "data.id", "sharding_function" => "pg_bigint_hash", "shards" => { - "0" => { "database" => "shard0", "servers" => [["localhost", primary0.port.to_s, "primary"]] }, - "1" => { "database" => "shard1", "servers" => [["localhost", primary1.port.to_s, "primary"]] }, - "2" => { "database" => "shard2", "servers" => [["localhost", primary2.port.to_s, "primary"]] }, + "0" => { "database" => "shard0", "servers" => [["localhost", primary0.port.to_i, "primary"]] }, + "1" => { "database" => "shard1", "servers" => [["localhost", primary1.port.to_i, "primary"]] }, + "2" => { "database" => "shard2", "servers" => [["localhost", primary2.port.to_i, "primary"]] }, }, "users" => { "0" => user }, "plugins" => { @@ -100,7 +100,7 @@ def self.single_instance_setup(pool_name, pool_size, pool_mode="transaction", lb "0" => { "database" => "shard0", "servers" => [ - ["localhost", primary.port.to_s, "primary"] + ["localhost", primary.port.to_i, "primary"] ] }, }, @@ -146,10 +146,10 @@ def self.single_shard_setup(pool_name, pool_size, pool_mode="transaction", lb_mo "0" => { "database" => "shard0", "servers" => [ - ["localhost", primary.port.to_s, "primary"], - ["localhost", replica0.port.to_s, "replica"], - ["localhost", replica1.port.to_s, "replica"], - ["localhost", replica2.port.to_s, "replica"] + ["localhost", primary.port.to_i, "primary"], + ["localhost", replica0.port.to_i, "replica"], + ["localhost", replica1.port.to_i, "replica"], + ["localhost", replica2.port.to_i, "replica"] ] }, }, diff --git a/tests/ruby/helpers/pgcat_process.rb b/tests/ruby/helpers/pgcat_process.rb index dd3fd0529..9328ff60b 100644 --- a/tests/ruby/helpers/pgcat_process.rb +++ b/tests/ruby/helpers/pgcat_process.rb @@ -1,8 +1,10 @@ require 'pg' -require 'toml' +require 'json' +require 'tempfile' require 'fileutils' require 'securerandom' +class ConfigReloadFailed < StandardError; end class PgcatProcess attr_reader :port attr_reader :pid @@ -18,7 +20,7 @@ def self.finalize(pid, log_filename, config_filename) end def initialize(log_level) - @env = {"RUST_LOG" => log_level} + @env = {} @port = rand(20000..32760) @log_level = log_level @log_filename = "/tmp/pgcat_log_#{SecureRandom.urlsafe_base64}.log" @@ -30,7 +32,7 @@ def initialize(log_level) '../../target/debug/pgcat' end - @command = "#{command_path} #{@config_filename}" + @command = "#{command_path} #{@config_filename} --log-level #{@log_level}" FileUtils.cp("../../pgcat.toml", @config_filename) cfg = current_config @@ -46,22 +48,34 @@ def logs def update_config(config_hash) @original_config = current_config - output_to_write = TOML::Generator.new(config_hash).body - output_to_write = output_to_write.gsub(/,\s*["|'](\d+)["|']\s*,/, ',\1,') - output_to_write = output_to_write.gsub(/,\s*["|'](\d+)["|']\s*\]/, ',\1]') - File.write(@config_filename, output_to_write) + Tempfile.create('json_out', '/tmp') do |f| + f.write(config_hash.to_json) + f.flush + `cat #{f.path} | yj -jt > #{@config_filename}` + end end def current_config - loadable_string = File.read(@config_filename) - loadable_string = loadable_string.gsub(/,\s*(\d+)\s*,/, ', "\1",') - loadable_string = loadable_string.gsub(/,\s*(\d+)\s*\]/, ', "\1"]') - TOML.load(loadable_string) + JSON.parse(`cat #{@config_filename} | yj -tj`) + end + + def raw_config_file + File.read(@config_filename) end def reload_config - `kill -s HUP #{@pid}` - sleep 0.5 + conn = PG.connect(admin_connection_string) + + conn.async_exec("RELOAD") + rescue PG::ConnectionBad => e + errors = logs.split("Reloading config").last + errors = errors.gsub(/\e\[([;\d]+)?m/, '') # Remove color codes + errors = errors. + split("\n").select{|line| line.include?("ERROR") }. + map { |line| line.split("pgcat::config: ").last } + raise ConfigReloadFailed, errors.join("\n") + ensure + conn&.close end def start @@ -116,11 +130,11 @@ def connection_string(pool_name, username, password = nil, parameters: {}) cfg = current_config user_idx, user_obj = cfg["pools"][pool_name]["users"].detect { |k, user| user["username"] == username } connection_string = "postgresql://#{username}:#{password || user_obj["password"]}@0.0.0.0:#{@port}/#{pool_name}" - + # Add the additional parameters to the connection string parameter_string = parameters.map { |key, value| "#{key}=#{value}" }.join("&") connection_string += "?#{parameter_string}" unless parameter_string.empty? - + connection_string end diff --git a/tests/ruby/mirrors_spec.rb b/tests/ruby/mirrors_spec.rb index 898d0d718..b6a4514ce 100644 --- a/tests/ruby/mirrors_spec.rb +++ b/tests/ruby/mirrors_spec.rb @@ -11,9 +11,9 @@ before do new_configs = processes.pgcat.current_config new_configs["pools"]["sharded_db"]["shards"]["0"]["mirrors"] = [ - [mirror_host, mirror_pg.port.to_s, "0"], - [mirror_host, mirror_pg.port.to_s, "0"], - [mirror_host, mirror_pg.port.to_s, "0"], + [mirror_host, mirror_pg.port.to_i, 0], + [mirror_host, mirror_pg.port.to_i, 0], + [mirror_host, mirror_pg.port.to_i, 0], ] processes.pgcat.update_config(new_configs) processes.pgcat.reload_config @@ -31,7 +31,8 @@ runs.times { conn.async_exec("SELECT 1 + 2") } sleep 0.5 expect(processes.all_databases.first.count_select_1_plus_2).to eq(runs) - expect(mirror_pg.count_select_1_plus_2).to eq(runs * 3) + # Allow some slack in mirroring successes + expect(mirror_pg.count_select_1_plus_2).to be > ((runs - 5) * 3) end context "when main server connection is closed" do @@ -42,9 +43,9 @@ new_configs = processes.pgcat.current_config new_configs["pools"]["sharded_db"]["idle_timeout"] = 5000 + i new_configs["pools"]["sharded_db"]["shards"]["0"]["mirrors"] = [ - [mirror_host, mirror_pg.port.to_s, "0"], - [mirror_host, mirror_pg.port.to_s, "0"], - [mirror_host, mirror_pg.port.to_s, "0"], + [mirror_host, mirror_pg.port.to_i, 0], + [mirror_host, mirror_pg.port.to_i, 0], + [mirror_host, mirror_pg.port.to_i, 0], ] processes.pgcat.update_config(new_configs) processes.pgcat.reload_config diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index 1d4ade4c9..aa17e8ec7 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -252,7 +252,7 @@ end expect(processes.primary.count_query("RESET ROLE")).to eq(10) - end + end end context "transaction mode" do @@ -317,7 +317,7 @@ conn.async_exec("SET statement_timeout to 1500") expect(conn.async_exec("SHOW statement_timeout")[0]["statement_timeout"]).to eq(orignal_statement_timeout) end - + end context "transaction mode with transactions" do @@ -354,7 +354,6 @@ conn.async_exec("SET statement_timeout TO 1000") conn.close - puts processes.pgcat.logs expect(processes.primary.count_query("RESET ALL")).to eq(0) end @@ -365,7 +364,6 @@ conn.close - puts processes.pgcat.logs expect(processes.primary.count_query("RESET ALL")).to eq(0) end end @@ -376,10 +374,9 @@ before do current_configs = processes.pgcat.current_config correct_idle_client_transaction_timeout = current_configs["general"]["idle_client_in_transaction_timeout"] - puts(current_configs["general"]["idle_client_in_transaction_timeout"]) - + current_configs["general"]["idle_client_in_transaction_timeout"] = 0 - + processes.pgcat.update_config(current_configs) # with timeout 0 processes.pgcat.reload_config end @@ -397,9 +394,9 @@ context "idle transaction timeout set to 500ms" do before do current_configs = processes.pgcat.current_config - correct_idle_client_transaction_timeout = current_configs["general"]["idle_client_in_transaction_timeout"] + correct_idle_client_transaction_timeout = current_configs["general"]["idle_client_in_transaction_timeout"] current_configs["general"]["idle_client_in_transaction_timeout"] = 500 - + processes.pgcat.update_config(current_configs) # with timeout 500 processes.pgcat.reload_config end @@ -418,7 +415,7 @@ conn.async_exec("BEGIN") conn.async_exec("SELECT 1") sleep(1) # above 500ms - expect{ conn.async_exec("COMMIT") }.to raise_error(PG::SystemError, /idle transaction timeout/) + expect{ conn.async_exec("COMMIT") }.to raise_error(PG::SystemError, /idle transaction timeout/) conn.async_exec("SELECT 1") # should be able to send another query conn.close end diff --git a/tests/ruby/sharding_spec.rb b/tests/ruby/sharding_spec.rb index 123c10dce..746627d1c 100644 --- a/tests/ruby/sharding_spec.rb +++ b/tests/ruby/sharding_spec.rb @@ -7,11 +7,11 @@ before do conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) - # Setup the sharding data 3.times do |i| conn.exec("SET SHARD TO '#{i}'") - conn.exec("DELETE FROM data WHERE id > 0") + + conn.exec("DELETE FROM data WHERE id > 0") rescue nil end 18.times do |i| @@ -19,10 +19,11 @@ conn.exec("SET SHARDING KEY TO '#{i}'") conn.exec("INSERT INTO data (id, value) VALUES (#{i}, 'value_#{i}')") end + + conn.close end after do - processes.all_databases.map(&:reset) processes.pgcat.shutdown end @@ -48,4 +49,148 @@ end end end + + describe "no_shard_specified_behavior config" do + context "when default shard number is invalid" do + it "prevents config reload" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + + current_configs = processes.pgcat.current_config + current_configs["pools"]["sharded_db"]["default_shard"] = "shard_99" + + processes.pgcat.update_config(current_configs) + + expect { processes.pgcat.reload_config }.to raise_error(ConfigReloadFailed, /Invalid shard 99/) + end + end + end + + describe "comment-based routing" do + context "when no configs are set" do + it "routes queries with a shard_id comment to the default shard" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + 10.times { conn.async_exec("/* shard_id: 2 */ SELECT 1 + 2") } + + expect(processes.all_databases.map(&:count_select_1_plus_2)).to eq([10, 0, 0]) + end + + it "does not honor no_shard_specified_behavior directives" do + end + end + + [ + ["shard_id_regex", "/\\* the_shard_id: (\\d+) \\*/", "/* the_shard_id: 1 */"], + ["sharding_key_regex", "/\\* the_sharding_key: (\\d+) \\*/", "/* the_sharding_key: 3 */"], + ].each do |config_name, config_value, comment_to_use| + context "when #{config_name} config is set" do + let(:no_shard_specified_behavior) { nil } + + before do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + + current_configs = processes.pgcat.current_config + current_configs["pools"]["sharded_db"][config_name] = config_value + if no_shard_specified_behavior + current_configs["pools"]["sharded_db"]["default_shard"] = no_shard_specified_behavior + else + current_configs["pools"]["sharded_db"].delete("default_shard") + end + + processes.pgcat.update_config(current_configs) + processes.pgcat.reload_config + end + + it "routes queries with a shard_id comment to the correct shard" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + 25.times { conn.async_exec("#{comment_to_use} SELECT 1 + 2") } + + expect(processes.all_databases.map(&:count_select_1_plus_2)).to eq([0, 25, 0]) + end + + context "when no_shard_specified_behavior config is set to random" do + let(:no_shard_specified_behavior) { "random" } + + context "with no shard comment" do + it "sends queries to random shard" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + 25.times { conn.async_exec("SELECT 1 + 2") } + + expect(processes.all_databases.map(&:count_select_1_plus_2).all?(&:positive?)).to be true + end + end + + context "with a shard comment" do + it "honors the comment" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + 25.times { conn.async_exec("#{comment_to_use} SELECT 1 + 2") } + + expect(processes.all_databases.map(&:count_select_1_plus_2)).to eq([0, 25, 0]) + end + end + end + + context "when no_shard_specified_behavior config is set to random_healthy" do + let(:no_shard_specified_behavior) { "random_healthy" } + + context "with no shard comment" do + it "sends queries to random healthy shard" do + + good_databases = [processes.all_databases[0], processes.all_databases[2]] + bad_database = processes.all_databases[1] + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + 250.times { conn.async_exec("SELECT 99") } + bad_database.take_down do + 250.times do + conn.async_exec("SELECT 99") + rescue PG::ConnectionBad => e + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + end + end + + # Routes traffic away from bad shard + 25.times { conn.async_exec("SELECT 1 + 2") } + expect(good_databases.map(&:count_select_1_plus_2).all?(&:positive?)).to be true + expect(bad_database.count_select_1_plus_2).to eq(0) + + # Routes traffic to the bad shard if the shard_id is specified + 25.times { conn.async_exec("#{comment_to_use} SELECT 1 + 2") } + bad_database = processes.all_databases[1] + expect(bad_database.count_select_1_plus_2).to eq(25) + end + end + + context "with a shard comment" do + it "honors the comment" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + 25.times { conn.async_exec("#{comment_to_use} SELECT 1 + 2") } + + expect(processes.all_databases.map(&:count_select_1_plus_2)).to eq([0, 25, 0]) + end + end + end + + context "when no_shard_specified_behavior config is set to shard_x" do + let(:no_shard_specified_behavior) { "shard_2" } + + context "with no shard comment" do + it "sends queries to the specified shard" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + 25.times { conn.async_exec("SELECT 1 + 2") } + + expect(processes.all_databases.map(&:count_select_1_plus_2)).to eq([0, 0, 25]) + end + end + + context "with a shard comment" do + it "honors the comment" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + 25.times { conn.async_exec("#{comment_to_use} SELECT 1 + 2") } + + expect(processes.all_databases.map(&:count_select_1_plus_2)).to eq([0, 25, 0]) + end + end + end + end + end + end end From 598e71b6ed2bcfeb99c6a534111d4ed303bf1f91 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 11 Sep 2023 16:39:11 -0700 Subject: [PATCH 463/551] User-friendly error messages (#586) --- src/client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client.rs b/src/client.rs index 4b2811216..dc0e2927b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -547,7 +547,7 @@ where error_response( &mut write, &format!( - "No pool configured for database: {:?}, user: {:?}", + "There is no database \"{}\" and user \"{}\" in our system. Create a free database on https://postgresml.org to get started.", pool_name, username ), ) @@ -1099,7 +1099,7 @@ where error_response( &mut self.write, - format!("could not get connection from the pool - {}", err).as_str(), + "Your database ran out of GPU concurrency capacity. Please upgrade by logging into your account on https://postgresml.org and increasing your GPU concurrency limit.", ) .await?; From 6d6f11c9304b6c8b165f508ca6d15726d34ba1f4 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 11 Sep 2023 16:39:31 -0700 Subject: [PATCH 464/551] Revert "User-friendly error messages" (#587) Revert "User-friendly error messages (#586)" This reverts commit 598e71b6ed2bcfeb99c6a534111d4ed303bf1f91. --- src/client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client.rs b/src/client.rs index dc0e2927b..4b2811216 100644 --- a/src/client.rs +++ b/src/client.rs @@ -547,7 +547,7 @@ where error_response( &mut write, &format!( - "There is no database \"{}\" and user \"{}\" in our system. Create a free database on https://postgresml.org to get started.", + "No pool configured for database: {:?}, user: {:?}", pool_name, username ), ) @@ -1099,7 +1099,7 @@ where error_response( &mut self.write, - "Your database ran out of GPU concurrency capacity. Please upgrade by logging into your account on https://postgresml.org and increasing your GPU concurrency limit.", + format!("could not get connection from the pool - {}", err).as_str(), ) .await?; From 92260d1e4d98c1d20bea028ccea7427313591a2d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 20 Sep 2023 09:11:16 -0700 Subject: [PATCH 465/551] Include `thread_id` in the logs (#592) Include `thread_id` in the logs. --- src/logger.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/logger.rs b/src/logger.rs index f9016ebfa..867b1a24b 100644 --- a/src/logger.rs +++ b/src/logger.rs @@ -8,6 +8,7 @@ pub fn init(args: &Args) { let filter = EnvFilter::from_default_env().add_directive(args.log_level.into()); let trace_sub = tracing_subscriber::fmt() + .with_thread_ids(true) .with_env_filter(filter) .with_ansi(!args.no_color); From de68c8c9dd974c45a804e45a0c5518fe0b8d009a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 20 Sep 2023 10:29:30 -0700 Subject: [PATCH 466/551] Add a cache layer to Docker for development (#594) * Add a cache layer to Docker. * Created a separate `dev` Docker file. * Fixed `Docker.dev` to build in non-release mode. --- Dockerfile.dev | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 Dockerfile.dev diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 000000000..a4b8d0ed0 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,25 @@ +FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef + +RUN apt-get update && \ + apt-get install -y build-essential + +WORKDIR /app + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +COPY --from=planner /app/recipe.json recipe.json +# Build dependencies - this is the caching Docker layer! +RUN cargo chef cook --release --recipe-path recipe.json +# Build application +COPY . . +RUN cargo build + +FROM debian:bookworm-slim +COPY --from=builder /app/target/release/pgcat /usr/bin/pgcat +COPY --from=builder /app/pgcat.toml /etc/pgcat/pgcat.toml +WORKDIR /etc/pgcat +ENV RUST_LOG=info +CMD ["pgcat"] From 666963d709e7e5e6450757fa893515ea41d84bff Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 21 Sep 2023 14:50:18 -0700 Subject: [PATCH 467/551] Report maxwait for clients that end up not getting a connection (#596) --- src/pool.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pool.rs b/src/pool.rs index 181234071..02dab2731 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -720,7 +720,7 @@ impl ConnectionPool { // since we last checked the server is ok. // Health checks are pretty expensive. if !require_healthcheck { - let checkout_time: u64 = now.elapsed().as_micros() as u64; + let checkout_time = now.elapsed().as_micros() as u64; client_stats.checkout_time(checkout_time); server .stats() @@ -734,7 +734,7 @@ impl ConnectionPool { .run_health_check(address, server, now, client_stats) .await { - let checkout_time: u64 = now.elapsed().as_micros() as u64; + let checkout_time = now.elapsed().as_micros() as u64; client_stats.checkout_time(checkout_time); server .stats() @@ -746,7 +746,12 @@ impl ConnectionPool { continue; } } + client_stats.idle(); + + let checkout_time = now.elapsed().as_micros() as u64; + client_stats.checkout_time(checkout_time); + Err(Error::AllServersDown) } From 0841fc862b48b240deb2b072060380a36fce0860 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 21 Sep 2023 15:55:22 -0700 Subject: [PATCH 468/551] Mark admin clients as disconnected on error (#597) --- src/client.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/client.rs b/src/client.rs index 4b2811216..2ec5b6a71 100644 --- a/src/client.rs +++ b/src/client.rs @@ -155,10 +155,10 @@ pub async fn client_entrypoint( if !client.is_admin() { let _ = drain.send(-1).await; + } - if result.is_err() { - client.stats.disconnect(); - } + if result.is_err() { + client.stats.disconnect(); } result @@ -207,10 +207,10 @@ pub async fn client_entrypoint( if !client.is_admin() { let _ = drain.send(-1).await; + } - if result.is_err() { - client.stats.disconnect(); - } + if result.is_err() { + client.stats.disconnect(); } result @@ -261,10 +261,10 @@ pub async fn client_entrypoint( if !client.is_admin() { let _ = drain.send(-1).await; + } - if result.is_err() { - client.stats.disconnect(); - } + if result.is_err() { + client.stats.disconnect(); } result @@ -290,11 +290,12 @@ pub async fn client_entrypoint( if !client.is_admin() { let _ = drain.send(-1).await; + } - if result.is_err() { - client.stats.disconnect(); - } + if result.is_err() { + client.stats.disconnect(); } + result } @@ -1811,7 +1812,6 @@ impl Drop for Client { // Dirty shutdown // TODO: refactor, this is not the best way to handle state management. - if self.connected_to_server && self.last_server_stats.is_some() { self.last_server_stats.as_ref().unwrap().idle(); } From b364bfc5fc04de858e613bf71987777679fbdc28 Mon Sep 17 00:00:00 2001 From: Kevin Elliott Date: Mon, 25 Sep 2023 18:45:51 -0700 Subject: [PATCH 469/551] Fix incorrect data output for plugin query_logger (#601) Update query_logger.rs Pool and user were incorrectly swapped and needed to be fixed. --- src/plugins/query_logger.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/query_logger.rs b/src/plugins/query_logger.rs index debdf39fa..012b040ed 100644 --- a/src/plugins/query_logger.rs +++ b/src/plugins/query_logger.rs @@ -31,7 +31,7 @@ impl<'a> Plugin for QueryLogger<'a> { .map(|q| q.to_string()) .collect::>() .join("; "); - info!("[pool: {}][user: {}] {}", self.user, self.db, query); + info!("[pool: {}][user: {}] {}", self.db, self.user, query); Ok(PluginOutput::Allow) } From 8d9927b0c248680170ade2c9c4352447f8e90826 Mon Sep 17 00:00:00 2001 From: Nicolas Vanelslande Date: Wed, 27 Sep 2023 17:03:39 +0100 Subject: [PATCH 470/551] Add psql to the container image. (#607) It could be used to implement container health checks. Example: PGPASSWORD="" psql -U pgcat -p 6432 -h 127.0.0.1 -tA -c "show version;" -d pgcat >/dev/null --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index f2d580629..be3df5365 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,12 @@ WORKDIR /app RUN cargo build --release FROM debian:bookworm-slim +RUN apt-get update && apt-get install -o Dpkg::Options::=--force-confdef -yq --no-install-recommends \ + postgresql-client \ + # Clean up layer + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && truncate -s 0 /var/log/*log COPY --from=builder /app/target/release/pgcat /usr/bin/pgcat COPY --from=builder /app/pgcat.toml /etc/pgcat/pgcat.toml WORKDIR /etc/pgcat From c702a3251a6e89fd6021519866cc7e7fc3da71e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:30:42 -0700 Subject: [PATCH 471/551] chore(deps): bump webpki from 0.22.0 to 0.22.2 in /tests/rust (#609) Bumps [webpki](https://github.com/briansmith/webpki) from 0.22.0 to 0.22.2. - [Commits](https://github.com/briansmith/webpki/commits) --- updated-dependencies: - dependency-name: webpki dependency-type: indirect ... Signed-off-by: dependabot[bot] --- tests/rust/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rust/Cargo.lock b/tests/rust/Cargo.lock index f24df81e0..b182ca9ef 100644 --- a/tests/rust/Cargo.lock +++ b/tests/rust/Cargo.lock @@ -1206,9 +1206,9 @@ dependencies = [ [[package]] name = "webpki" -version = "0.22.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f" dependencies = [ "ring", "untrusted", From e9960e8609f1576b3fcf36a4bce1188772444684 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 3 Oct 2023 09:36:13 -0700 Subject: [PATCH 472/551] Automatic sharding for INSERT, UPDATE, and DELETE statements. (#610) Added support for INSERT, UPDATE, and DELETE for auto-sharding. --- src/query_router.rs | 609 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 507 insertions(+), 102 deletions(-) diff --git a/src/query_router.rs b/src/query_router.rs index 9d7a106a4..189f2dcc6 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -4,10 +4,10 @@ use bytes::{Buf, BytesMut}; use log::{debug, error}; use once_cell::sync::OnceCell; use regex::{Regex, RegexSet}; -use sqlparser::ast::Statement::{Query, StartTransaction}; +use sqlparser::ast::Statement::{Delete, Insert, Query, StartTransaction, Update}; use sqlparser::ast::{ - BinaryOperator, Expr, Ident, JoinConstraint, JoinOperator, SetExpr, Statement, TableFactor, - Value, + Assignment, BinaryOperator, Expr, Ident, JoinConstraint, JoinOperator, SetExpr, Statement, + TableFactor, TableWithJoins, Value, }; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; @@ -403,6 +403,9 @@ impl QueryRouter { return Err(Error::QueryRouterParserError("empty query".into())); } + let mut visited_write_statement = false; + let mut prev_inferred_shard = None; + for q in ast { match q { // All transactions go to the primary, probably a write. @@ -420,29 +423,38 @@ impl QueryRouter { // or discard shard selection. If they point to the same shard though, // we can let them through as-is. // This is basically building a database now :) - match self.infer_shard(query) { - Some(shard) => { - self.active_shard = Some(shard); - debug!("Automatically using shard: {:?}", self.active_shard); - } - - None => (), - }; + let inferred_shard = self.infer_shard(query); + self.handle_inferred_shard(inferred_shard, &mut prev_inferred_shard)?; } None => (), }; - self.active_role = match self.primary_reads_enabled() { - false => Some(Role::Replica), // If primary should not be receiving reads, use a replica. - true => None, // Any server role is fine in this case. + // If we already visited a write statement, we should be going to the primary. + if !visited_write_statement { + self.active_role = match self.primary_reads_enabled() { + false => Some(Role::Replica), // If primary should not be receiving reads, use a replica. + true => None, // Any server role is fine in this case. + } } } // Likely a write _ => { + match &self.pool_settings.automatic_sharding_key { + Some(_) => { + // TODO: similar to the above, if we have multiple queries in the + // same message, we can either split them and execute them individually + // or discard shard selection. If they point to the same shard though, + // we can let them through as-is. + let inferred_shard = self.infer_shard_on_write(q)?; + self.handle_inferred_shard(inferred_shard, &mut prev_inferred_shard)?; + } + + None => (), + }; + visited_write_statement = true; self.active_role = Some(Role::Primary); - break; } }; } @@ -450,6 +462,208 @@ impl QueryRouter { Ok(()) } + fn handle_inferred_shard( + &mut self, + inferred_shard: Option, + prev_inferred_shard: &mut Option, + ) -> Result<(), Error> { + match inferred_shard { + Some(shard) => { + if let Some(prev_shard) = *prev_inferred_shard { + if prev_shard != shard { + debug!("Found more than one shard in the query, not supported yet"); + return Err(Error::QueryRouterParserError( + "multiple shards in query".into(), + )); + } + } + *prev_inferred_shard = Some(shard); + self.active_shard = Some(shard); + debug!("Automatically using shard: {:?}", self.active_shard); + } + + None => (), + }; + Ok(()) + } + + fn infer_shard_on_write(&mut self, q: &Statement) -> Result, Error> { + let mut exprs = Vec::new(); + + // Collect all table names from the query. + let mut table_names = Vec::new(); + + match q { + Insert { + or, + into: _, + table_name, + columns, + overwrite: _, + source, + partitioned, + after_columns, + table: _, + on: _, + returning: _, + } => { + // Not supported in postgres. + assert!(or.is_none()); + assert!(partitioned.is_none()); + assert!(after_columns.is_empty()); + + Self::process_table(table_name, &mut table_names); + Self::process_query(&*source, &mut exprs, &mut table_names, &Some(columns)); + } + Delete { + tables, + from, + using, + selection, + returning: _, + } => { + if let Some(expr) = selection { + exprs.push(expr.clone()); + } + + // Multi tables delete are not supported in postgres. + assert!(tables.is_empty()); + + Self::process_tables_with_join(&from, &mut exprs, &mut table_names); + if let Some(using_tbl_with_join) = using { + Self::process_tables_with_join( + using_tbl_with_join, + &mut exprs, + &mut table_names, + ); + } + Self::process_selection(selection, &mut exprs); + } + Update { + table, + assignments, + from, + selection, + returning: _, + } => { + Self::process_table_with_join(table, &mut exprs, &mut table_names); + if let Some(from_tbl) = from { + Self::process_table_with_join(from_tbl, &mut exprs, &mut table_names); + } + Self::process_selection(selection, &mut exprs); + self.assignment_parser(assignments)?; + } + _ => { + return Ok(None); + } + }; + + Ok(self.infer_shard_from_exprs(exprs, table_names)) + } + + fn process_query( + query: &sqlparser::ast::Query, + exprs: &mut Vec, + table_names: &mut Vec>, + columns: &Option<&Vec>, + ) { + match &*query.body { + SetExpr::Query(query) => { + Self::process_query(&*query, exprs, table_names, columns); + } + + // SELECT * FROM ... + // We understand that pretty well. + SetExpr::Select(select) => { + Self::process_tables_with_join(&select.from, exprs, table_names); + + // Parse the actual "FROM ..." + Self::process_selection(&select.selection, exprs); + } + + SetExpr::Values(values) => { + if let Some(cols) = columns { + for row in values.rows.iter() { + for (i, expr) in row.iter().enumerate() { + if cols.len() > i { + exprs.push(Expr::BinaryOp { + left: Box::new(Expr::Identifier(cols[i].clone())), + op: BinaryOperator::Eq, + right: Box::new(expr.clone()), + }); + } + } + } + } + } + _ => (), + }; + } + + fn process_selection(selection: &Option, exprs: &mut Vec) { + match selection { + Some(selection) => { + exprs.push(selection.clone()); + } + + None => (), + }; + } + + fn process_tables_with_join( + tables: &Vec, + exprs: &mut Vec, + table_names: &mut Vec>, + ) { + for table in tables.iter() { + Self::process_table_with_join(table, exprs, table_names); + } + } + + fn process_table_with_join( + table: &TableWithJoins, + exprs: &mut Vec, + table_names: &mut Vec>, + ) { + match &table.relation { + TableFactor::Table { name, .. } => { + Self::process_table(name, table_names); + } + + _ => (), + }; + + // Get table names from all the joins. + for join in table.joins.iter() { + match &join.relation { + TableFactor::Table { name, .. } => { + Self::process_table(name, table_names); + } + + _ => (), + }; + + // We can filter results based on join conditions, e.g. + // SELECT * FROM t INNER JOIN B ON B.sharding_key = 5; + match &join.join_operator { + JoinOperator::Inner(inner_join) => match &inner_join { + JoinConstraint::On(expr) => { + // Parse the selection criteria later. + exprs.push(expr.clone()); + } + + _ => (), + }, + + _ => (), + }; + } + } + + fn process_table(name: &sqlparser::ast::ObjectName, table_names: &mut Vec>) { + table_names.push(name.0.clone()) + } + /// Parse the shard number from the Bind message /// which contains the arguments for a prepared statement. /// @@ -592,6 +806,33 @@ impl QueryRouter { } } + /// An `assignments` exists in the `UPDATE` statements. This parses the assignments and makes + /// sure that we are not updating the sharding key. It's not supported yet. + fn assignment_parser(&self, assignments: &Vec) -> Result<(), Error> { + let sharding_key = self + .pool_settings + .automatic_sharding_key + .as_ref() + .unwrap() + .split(".") + .map(|ident| Ident::new(ident.to_lowercase())) + .collect::>(); + + // Sharding key must be always fully qualified + assert_eq!(sharding_key.len(), 2); + + for a in assignments { + if sharding_key[0].value == "*" { + if sharding_key[1].value == a.id.last().unwrap().value.to_lowercase() { + return Err(Error::QueryRouterParserError( + "Sharding key cannot be updated.".into(), + )); + } + } + } + Ok(()) + } + /// A `selection` is the `WHERE` clause. This parses /// the clause and extracts the sharding key, if present. fn selection_parser(&self, expr: &Expr, table_names: &Vec>) -> Vec { @@ -604,7 +845,7 @@ impl QueryRouter { .as_ref() .unwrap() .split(".") - .map(|ident| Ident::new(ident)) + .map(|ident| Ident::new(ident.to_lowercase())) .collect::>(); // Sharding key must be always fully qualified @@ -620,7 +861,7 @@ impl QueryRouter { Expr::Identifier(ident) => { // Only if we're dealing with only one table // and there is no ambiguity - if &ident.value == &sharding_key[1].value { + if &ident.value.to_lowercase() == &sharding_key[1].value { // Sharding key is unique enough, don't worry about // table names. if &sharding_key[0].value == "*" { @@ -633,13 +874,13 @@ impl QueryRouter { // SELECT * FROM t WHERE sharding_key = 5 // Make sure the table name from the sharding key matches // the table name from the query. - found = &sharding_key[0].value == &table[0].value; + found = &sharding_key[0].value == &table[0].value.to_lowercase(); } else if table.len() == 2 { // Table name is fully qualified with the schema: e.g. // SELECT * FROM public.t WHERE sharding_key = 5 // Ignore the schema (TODO: at some point, we want schema support) // and use the table name only. - found = &sharding_key[0].value == &table[1].value; + found = &sharding_key[0].value == &table[1].value.to_lowercase(); } else { debug!("Got table name with more than two idents, which is not possible"); } @@ -651,8 +892,9 @@ impl QueryRouter { // The key is fully qualified in the query, // it will exist or Postgres will throw an error. if idents.len() == 2 { - found = &sharding_key[0].value == &idents[0].value - && &sharding_key[1].value == &idents[1].value; + found = (&sharding_key[0].value == "*" + || &sharding_key[0].value == &idents[0].value.to_lowercase()) + && &sharding_key[1].value == &idents[1].value.to_lowercase(); } // TODO: key can have schema as well, e.g. public.data.id (len == 3) } @@ -705,100 +947,48 @@ impl QueryRouter { /// Try to figure out which shard the query should go to. fn infer_shard(&mut self, query: &sqlparser::ast::Query) -> Option { - let mut shards = BTreeSet::new(); let mut exprs = Vec::new(); - match &*query.body { - SetExpr::Query(query) => { - match self.infer_shard(&*query) { - Some(shard) => { - shards.insert(shard); - } - None => (), - }; - } + // Collect all table names from the query. + let mut table_names = Vec::new(); - // SELECT * FROM ... - // We understand that pretty well. - SetExpr::Select(select) => { - // Collect all table names from the query. - let mut table_names = Vec::new(); - - for table in select.from.iter() { - match &table.relation { - TableFactor::Table { name, .. } => { - table_names.push(name.0.clone()); - } - - _ => (), - }; + Self::process_query(query, &mut exprs, &mut table_names, &None); + self.infer_shard_from_exprs(exprs, table_names) + } - // Get table names from all the joins. - for join in table.joins.iter() { - match &join.relation { - TableFactor::Table { name, .. } => { - table_names.push(name.0.clone()); - } + fn infer_shard_from_exprs( + &mut self, + exprs: Vec, + table_names: Vec>, + ) -> Option { + let mut shards = BTreeSet::new(); - _ => (), - }; + let sharder = Sharder::new( + self.pool_settings.shards, + self.pool_settings.sharding_function, + ); - // We can filter results based on join conditions, e.g. - // SELECT * FROM t INNER JOIN B ON B.sharding_key = 5; - match &join.join_operator { - JoinOperator::Inner(inner_join) => match &inner_join { - JoinConstraint::On(expr) => { - // Parse the selection criteria later. - exprs.push(expr.clone()); - } + // Look for sharding keys in either the join condition + // or the selection. + for expr in exprs.iter() { + let sharding_keys = self.selection_parser(expr, &table_names); - _ => (), - }, + // TODO: Add support for prepared statements here. + // This should just give us the position of the value in the `B` message. - _ => (), - }; + for value in sharding_keys { + match value { + ShardingKey::Value(value) => { + let shard = sharder.shard(value); + shards.insert(shard); } - } - // Parse the actual "FROM ..." - match &select.selection { - Some(selection) => { - exprs.push(selection.clone()); + ShardingKey::Placeholder(position) => { + self.placeholders.push(position); } - - None => (), }; - - let sharder = Sharder::new( - self.pool_settings.shards, - self.pool_settings.sharding_function, - ); - - // Look for sharding keys in either the join condition - // or the selection. - for expr in exprs.iter() { - let sharding_keys = self.selection_parser(expr, &table_names); - - // TODO: Add support for prepared statements here. - // This should just give us the position of the value in the `B` message. - - for value in sharding_keys { - match value { - ShardingKey::Value(value) => { - let shard = sharder.shard(value); - shards.insert(shard); - } - - ShardingKey::Placeholder(position) => { - self.placeholders.push(position); - } - }; - } - } } - _ => (), - }; - + } match shards.len() { // Didn't find a sharding key, you're on your own. 0 => { @@ -1414,6 +1604,221 @@ mod test { assert_eq!(qr.shard().unwrap(), 0); } + fn auto_shard_wrapper(qry: &str, should_succeed: bool) -> Option { + let mut qr = QueryRouter::new(); + qr.pool_settings.automatic_sharding_key = Some("*.w_id".to_string()); + qr.pool_settings.shards = 3; + qr.pool_settings.query_parser_read_write_splitting = true; + assert_eq!(qr.shard(), None); + let infer_res = qr.infer(&qr.parse(&simple_query(qry)).unwrap()); + assert_eq!(infer_res.is_ok(), should_succeed); + qr.shard() + } + + fn auto_shard(qry: &str) -> Option { + auto_shard_wrapper(qry, true) + } + + fn auto_shard_fails(qry: &str) -> Option { + auto_shard_wrapper(qry, false) + } + + #[test] + fn test_automatic_sharding_insert_update_delete() { + QueryRouter::setup(); + + assert_eq!( + auto_shard_fails( + "UPDATE ORDERS SET w_id = 3 WHERE O_ID = 3 AND O_D_ID = 3 AND W_ID = 5" + ), + None + ); + + assert_eq!( + auto_shard_fails( + "UPDATE ORDERS o SET o.W_ID = 3 WHERE o.O_ID = 3 AND o.O_D_ID = 3 AND o.W_ID = 5" + ), + None + ); + + assert_eq!( + auto_shard( + "UPDATE ORDERS o SET o.O_CARRIER_ID = 3 WHERE o.O_ID = 3 AND o.O_D_ID = 3 AND o.W_ID = 5" + ), + Some(2) + ); + } + + #[test] + fn test_automatic_sharding_key_tpcc() { + QueryRouter::setup(); + + assert_eq!(auto_shard("SELECT * FROM my_tbl WHERE w_id = 5"), Some(2)); + assert_eq!( + auto_shard("BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ"), + None + ); + assert_eq!(auto_shard("COMMIT"), None); + assert_eq!(auto_shard("ROLLBACK"), None); + + assert_eq!(auto_shard("SELECT NO_O_ID FROM NEW_ORDER WHERE NO_D_ID = 7 AND W_ID = 5 AND NO_O_ID > 3 LIMIT 3"), Some(2)); + assert_eq!(auto_shard("SELECT NO_O_ID FROM NEW_ORDER no WHERE no.NO_D_ID = 7 AND no.W_ID = 5 AND no.NO_O_ID > 3 LIMIT 3"), Some(2)); + + assert_eq!( + auto_shard("DELETE FROM NEW_ORDER WHERE NO_D_ID = 7 AND W_ID = 5 AND NO_O_ID = 3"), + Some(2) + ); + + assert_eq!( + auto_shard("SELECT O_C_ID FROM ORDERS WHERE O_ID = 3 AND O_D_ID = 3 AND W_ID = 5"), + Some(2) + ); + assert_eq!( + auto_shard( + "UPDATE ORDERS SET O_CARRIER_ID = 3 WHERE O_ID = 3 AND O_D_ID = 3 AND W_ID = 5" + ), + Some(2) + ); + + assert_eq!( + auto_shard("UPDATE ORDER_LINE SET OL_DELIVERY_D = 3 WHERE OL_O_ID = 3 AND OL_D_ID = 3 AND W_ID = 5"), + Some(2) + ); + + assert_eq!( + auto_shard("SELECT SUM(OL_AMOUNT) FROM ORDER_LINE WHERE OL_O_ID = 3 AND OL_D_ID = 3 AND W_ID = 5"), + Some(2) + ); + + assert_eq!( + auto_shard("UPDATE CUSTOMER SET C_BALANCE = C_BALANCE + 3 WHERE C_ID = 3 AND C_D_ID = 3 AND W_ID = 5"), + Some(2) + ); + + assert_eq!( + auto_shard("SELECT W_TAX FROM WAREHOUSE WHERE W_ID = 5"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT D_TAX, D_NEXT_O_ID FROM DISTRICT WHERE D_ID = 3 AND W_ID = 5"), + Some(2) + ); + assert_eq!( + auto_shard("UPDATE DISTRICT SET D_NEXT_O_ID = 3 WHERE D_ID = 3 AND W_ID = 5"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT C_DISCOUNT, C_LAST, C_CREDIT FROM CUSTOMER WHERE W_ID = 5 AND C_D_ID = 3 AND C_ID = 3"), + Some(2) + ); + assert_eq!( + auto_shard("INSERT INTO ORDERS (O_ID, O_D_ID, W_ID, O_C_ID, O_ENTRY_D, O_CARRIER_ID, O_OL_CNT, O_ALL_LOCAL) VALUES (3, 3, 5, 3, 3, 3, 3, 3)"), + Some(2) + ); + assert_eq!( + auto_shard("INSERT INTO NEW_ORDER (NO_O_ID, NO_D_ID, W_ID) VALUES (3, 3, 5)"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT I_PRICE, I_NAME, I_DATA FROM ITEM WHERE I_ID = 3"), + None + ); + assert_eq!( + auto_shard("SELECT S_QUANTITY, S_DATA, S_YTD, S_ORDER_CNT, S_REMOTE_CNT, S_DIST_03 FROM STOCK WHERE S_I_ID = 3 AND W_ID = 5"), + Some(2) + ); + assert_eq!( + auto_shard("UPDATE STOCK SET S_QUANTITY = 3, S_YTD = 3, S_ORDER_CNT = 3, S_REMOTE_CNT = 3 WHERE S_I_ID = 3 AND W_ID = 5"), + Some(2) + ); + assert_eq!( + auto_shard("INSERT INTO ORDER_LINE (OL_O_ID, OL_D_ID, W_ID, OL_NUMBER, OL_I_ID, OL_SUPPLY_W_ID, OL_DELIVERY_D, OL_QUANTITY, OL_AMOUNT, OL_DIST_INFO) VALUES (3, 3, 5, 3, 3, 3, 3, 3, 3, 3)"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE W_ID = 5 AND C_D_ID = 3 AND C_ID = 3"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_BALANCE FROM CUSTOMER WHERE W_ID = 5 AND C_D_ID = 3 AND C_LAST = '3' ORDER BY C_FIRST"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT O_ID, O_CARRIER_ID, O_ENTRY_D FROM ORDERS WHERE W_ID = 5 AND O_D_ID = 3 AND O_C_ID = 3 ORDER BY O_ID DESC LIMIT 3"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT OL_SUPPLY_W_ID, OL_I_ID, OL_QUANTITY, OL_AMOUNT, OL_DELIVERY_D FROM ORDER_LINE WHERE W_ID = 5 AND OL_D_ID = 3 AND OL_O_ID = 3"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT W_NAME, W_STREET_1, W_STREET_2, W_CITY, W_STATE, W_ZIP FROM WAREHOUSE WHERE W_ID = 5"), + Some(2) + ); + assert_eq!( + auto_shard("UPDATE WAREHOUSE SET W_YTD = W_YTD + 3 WHERE W_ID = 5"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT D_NAME, D_STREET_1, D_STREET_2, D_CITY, D_STATE, D_ZIP FROM DISTRICT WHERE W_ID = 5 AND D_ID = 3"), + Some(2) + ); + assert_eq!( + auto_shard("UPDATE DISTRICT SET D_YTD = D_YTD + 3 WHERE W_ID = 5 AND D_ID = 3"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE W_ID = 5 AND C_D_ID = 3 AND C_ID = 3"), + Some(2) + ); + assert_eq!( + auto_shard("SELECT C_ID, C_FIRST, C_MIDDLE, C_LAST, C_STREET_1, C_STREET_2, C_CITY, C_STATE, C_ZIP, C_PHONE, C_SINCE, C_CREDIT, C_CREDIT_LIM, C_DISCOUNT, C_BALANCE, C_YTD_PAYMENT, C_PAYMENT_CNT, C_DATA FROM CUSTOMER WHERE W_ID = 5 AND C_D_ID = 3 AND C_LAST = '3' ORDER BY C_FIRST"), + Some(2) + ); + assert_eq!( + auto_shard("UPDATE CUSTOMER SET C_BALANCE = 3, C_YTD_PAYMENT = 3, C_PAYMENT_CNT = 3, C_DATA = 3 WHERE W_ID = 5 AND C_D_ID = 3 AND C_ID = 3"), + Some(2) + ); + assert_eq!( + auto_shard("UPDATE CUSTOMER SET C_BALANCE = 3, C_YTD_PAYMENT = 3, C_PAYMENT_CNT = 3 WHERE W_ID = 5 AND C_D_ID = 3 AND C_ID = 3"), + Some(2) + ); + + assert_eq!(auto_shard("INSERT INTO HISTORY (H_C_ID, H_C_D_ID, H_C_W_ID, H_D_ID, W_ID, H_DATE, H_AMOUNT, H_DATA) VALUES (3, 3, 5, 3, 5, 3, 3, 3)"), Some(2)); + assert_eq!( + auto_shard("SELECT D_NEXT_O_ID FROM DISTRICT WHERE W_ID = 5 AND D_ID = 3"), + Some(2) + ); + assert_eq!( + auto_shard( + "SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK + WHERE ORDER_LINE.W_ID = 5 + AND OL_D_ID = 3 + AND OL_O_ID < 3 + AND OL_O_ID >= 3 + AND STOCK.W_ID = 5 + AND S_I_ID = OL_I_ID + AND S_QUANTITY < 3" + ), + Some(2) + ); + + // This is a distributed query and contains two shards + assert_eq!( + auto_shard( + "SELECT COUNT(DISTINCT(OL_I_ID)) FROM ORDER_LINE, STOCK + WHERE ORDER_LINE.W_ID = 5 + AND OL_D_ID = 3 + AND OL_O_ID < 3 + AND OL_O_ID >= 3 + AND STOCK.W_ID = 7 + AND S_I_ID = OL_I_ID + AND S_QUANTITY < 3" + ), + None + ); + } + #[test] fn test_prepared_statements() { let stmt = "SELECT * FROM data WHERE id = $1"; From 37d6bc0bc5089cd42b96b8dff65af93c5270129e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 3 Oct 2023 13:13:21 -0700 Subject: [PATCH 473/551] Added a `Plugin` trait (#536) * Improved logging * Improved logging for more `Address` usages * Fixed lint issues. * Reverted the `Address` logging changes. * Applied the PR comment by @levkk. * Applied the PR comment by @levkk. * Applied the PR comment by @levkk. * Applied the PR comment by @levkk. --- src/config.rs | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/config.rs b/src/config.rs index 0404abc9d..90b4beb3b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -867,15 +867,26 @@ pub struct Plugins { pub prewarmer: Option, } +pub trait Plugin { + fn is_enabled(&self) -> bool; +} + impl std::fmt::Display for Plugins { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn is_enabled(arg: Option<&T>) -> bool { + if let Some(ref arg) = arg { + arg.is_enabled() + } else { + false + } + } write!( f, "interceptor: {}, table_access: {}, query_logger: {}, prewarmer: {}", - self.intercept.is_some(), - self.table_access.is_some(), - self.query_logger.is_some(), - self.prewarmer.is_some(), + is_enabled(self.intercept.as_ref()), + is_enabled(self.table_access.as_ref()), + is_enabled(self.query_logger.as_ref()), + is_enabled(self.prewarmer.as_ref()), ) } } @@ -886,23 +897,47 @@ pub struct Intercept { pub queries: BTreeMap, } +impl Plugin for Intercept { + fn is_enabled(&self) -> bool { + self.enabled + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, Hash, Eq)] pub struct TableAccess { pub enabled: bool, pub tables: Vec, } +impl Plugin for TableAccess { + fn is_enabled(&self) -> bool { + self.enabled + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, Hash, Eq)] pub struct QueryLogger { pub enabled: bool, } +impl Plugin for QueryLogger { + fn is_enabled(&self) -> bool { + self.enabled + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default, Hash, Eq)] pub struct Prewarmer { pub enabled: bool, pub queries: Vec, } +impl Plugin for Prewarmer { + fn is_enabled(&self) -> bool { + self.enabled + } +} + impl Intercept { pub fn substitute(&mut self, db: &str, user: &str) { for (_, query) in self.queries.iter_mut() { From 0071f75c99a7af7a8da987d9c4e8d35e57453e7b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Thu, 5 Oct 2023 18:13:22 -0700 Subject: [PATCH 474/551] Added `yj` to dev Dockerfile (#612) --- dev/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/Dockerfile b/dev/Dockerfile index bd24174f8..c3761b75c 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,6 +1,8 @@ FROM rust:1.70-bullseye # Dependencies +COPY --from=sclevine/yj /bin/yj /bin/yj +RUN /bin/yj -h RUN apt-get update -y \ && apt-get install -y \ llvm-11 psmisc postgresql-contrib postgresql-client \ From 7eb58d38889e998e5e3295724c70d0d532f2e85f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Tue, 10 Oct 2023 09:18:21 -0700 Subject: [PATCH 475/551] Added `clippy` to CI and fixed all `clippy` warnings (#613) * Fixed all clippy warnings. * Added `clippy` to CI. * Reverted an unwanted change + Applied `cargo fmt`. * Fixed the idiom version. * Revert "Fixed the idiom version." This reverts commit 6f78be0d4281de16e6ec123d6139c39d1a2c91ab. * Fixed clippy issues on CI. * Revert "Fixed clippy issues on CI." This reverts commit a9fa6ba1894ff94b3e1c09e3e8c498df76a7da11. * Revert "Reverted an unwanted change + Applied `cargo fmt`." This reverts commit 6bd37b6479cae0f6b0128bfe322487489e035c24. * Revert "Fixed all clippy warnings." This reverts commit d1f3b847e3da6c1caf281baaf6afc96b997b4165. * Removed Clippy * Removed Lint * `admin.rs` clippy fixes. * Applied more clippy changes. * Even more clippy changes. * `client.rs` clippy fixes. * `server.rs` clippy fixes. * Revert "Removed Lint" This reverts commit cb5042b144b71028a618a832bb49282ec9748cb0. * Revert "Removed Clippy" This reverts commit 6dec8bffb1aed4206eaf253e8af631fc66af7531. * Applied lint. * Revert "Revert "Fixed clippy issues on CI."" This reverts commit 49164a733cfb57a1e59f2f6cd50366e1863b697b. --- .circleci/config.yml | 3 + CONTRIBUTING.md | 2 +- src/admin.rs | 7 +- src/client.rs | 39 ++++---- src/cmd_args.rs | 2 +- src/config.rs | 103 +++++++++------------ src/messages.rs | 104 ++++++++++----------- src/mirrors.rs | 10 +- src/plugins/intercept.rs | 2 +- src/plugins/mod.rs | 1 + src/plugins/prewarmer.rs | 2 +- src/plugins/table_access.rs | 2 +- src/pool.rs | 42 ++++----- src/query_router.rs | 178 +++++++++++++++++------------------- src/scram.rs | 14 +-- src/server.rs | 35 +++---- src/sharding.rs | 6 +- src/stats/pool.rs | 10 +- 18 files changed, 258 insertions(+), 304 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c8344911f..072241124 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,9 @@ jobs: - run: name: "Lint" command: "cargo fmt --check" + - run: + name: "Clippy" + command: "cargo clippy --all --all-targets -- -Dwarnings" - run: name: "Tests" command: "cargo clean && cargo build && cargo test && bash .circleci/run_tests.sh && .circleci/generate_coverage.sh" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10d4924a7..e0d5d160f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ Thank you for contributing! Just a few tips here: -1. `cargo fmt` your code before opening up a PR +1. `cargo fmt` and `cargo clippy` your code before opening up a PR 2. Run the test suite (e.g. `pgbench`) to make sure everything still works. The tests are in `.circleci/run_tests.sh`. 3. Performance is important, make sure there are no regressions in your branch vs. `main`. diff --git a/src/admin.rs b/src/admin.rs index da9252920..f1b0c63fa 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -283,7 +283,7 @@ where { let mut res = BytesMut::new(); - let detail_msg = vec![ + let detail_msg = [ "", "SHOW HELP|CONFIG|DATABASES|POOLS|CLIENTS|SERVERS|USERS|VERSION", // "SHOW PEERS|PEER_POOLS", // missing PEERS|PEER_POOLS @@ -301,7 +301,6 @@ where // "KILL ", // "SUSPEND", "SHUTDOWN", - // "WAIT_CLOSE []", // missing ]; res.put(notify("Console usage", detail_msg.join("\n\t"))); @@ -802,7 +801,7 @@ where T: tokio::io::AsyncWrite + std::marker::Unpin, { let parts: Vec<&str> = match tokens.len() == 2 { - true => tokens[1].split(",").map(|part| part.trim()).collect(), + true => tokens[1].split(',').map(|part| part.trim()).collect(), false => Vec::new(), }; @@ -865,7 +864,7 @@ where T: tokio::io::AsyncWrite + std::marker::Unpin, { let parts: Vec<&str> = match tokens.len() == 2 { - true => tokens[1].split(",").map(|part| part.trim()).collect(), + true => tokens[1].split(',').map(|part| part.trim()).collect(), false => Vec::new(), }; diff --git a/src/client.rs b/src/client.rs index 2ec5b6a71..98a0669c5 100644 --- a/src/client.rs +++ b/src/client.rs @@ -131,7 +131,7 @@ pub async fn client_entrypoint( // Client requested a TLS connection. Ok((ClientConnectionType::Tls, _)) => { // TLS settings are configured, will setup TLS now. - if tls_certificate != None { + if tls_certificate.is_some() { debug!("Accepting TLS request"); let mut yes = BytesMut::new(); @@ -448,7 +448,7 @@ where None => "pgcat", }; - let client_identifier = ClientIdentifier::new(&application_name, &username, &pool_name); + let client_identifier = ClientIdentifier::new(application_name, username, pool_name); let admin = ["pgcat", "pgbouncer"] .iter() @@ -795,7 +795,7 @@ where let mut will_prepare = false; let client_identifier = ClientIdentifier::new( - &self.server_parameters.get_application_name(), + self.server_parameters.get_application_name(), &self.username, &self.pool_name, ); @@ -982,15 +982,11 @@ where } // Check on plugin results. - match plugin_output { - Some(PluginOutput::Deny(error)) => { - self.buffer.clear(); - error_response(&mut self.write, &error).await?; - plugin_output = None; - continue; - } - - _ => (), + if let Some(PluginOutput::Deny(error)) = plugin_output { + self.buffer.clear(); + error_response(&mut self.write, &error).await?; + plugin_output = None; + continue; }; // Check if the pool is paused and wait until it's resumed. @@ -1267,7 +1263,7 @@ where // Safe to unwrap because we know this message has a certain length and has the code // This reads the first byte without advancing the internal pointer and mutating the bytes - let code = *message.get(0).unwrap() as char; + let code = *message.first().unwrap() as char; trace!("Message: {}", code); @@ -1325,7 +1321,7 @@ where self.stats.transaction(); server .stats() - .transaction(&self.server_parameters.get_application_name()); + .transaction(self.server_parameters.get_application_name()); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -1400,13 +1396,10 @@ where let close: Close = (&message).try_into()?; if close.is_prepared_statement() && !close.anonymous() { - match self.prepared_statements.get(&close.name) { - Some(parse) => { - server.will_close(&parse.generated_name); - } - + if let Some(parse) = self.prepared_statements.get(&close.name) { + server.will_close(&parse.generated_name); + } else { // A prepared statement slipped through? Not impossible, since we don't support PREPARE yet. - None => (), }; } } @@ -1445,7 +1438,7 @@ where self.buffer.put(&message[..]); - let first_message_code = (*self.buffer.get(0).unwrap_or(&0)) as char; + let first_message_code = (*self.buffer.first().unwrap_or(&0)) as char; // Almost certainly true if first_message_code == 'P' && !prepared_statements_enabled { @@ -1477,7 +1470,7 @@ where self.stats.transaction(); server .stats() - .transaction(&self.server_parameters.get_application_name()); + .transaction(self.server_parameters.get_application_name()); // Release server back to the pool if we are in transaction mode. // If we are in session mode, we keep the server until the client disconnects. @@ -1739,7 +1732,7 @@ where client_stats.query(); server.stats().query( Instant::now().duration_since(query_start).as_millis() as u64, - &self.server_parameters.get_application_name(), + self.server_parameters.get_application_name(), ); Ok(()) diff --git a/src/cmd_args.rs b/src/cmd_args.rs index 3989d670f..1abb7ed94 100644 --- a/src/cmd_args.rs +++ b/src/cmd_args.rs @@ -25,7 +25,7 @@ pub struct Args { } pub fn parse() -> Args { - return Args::parse(); + Args::parse() } #[derive(ValueEnum, Clone, Debug)] diff --git a/src/config.rs b/src/config.rs index 90b4beb3b..f91e488e2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -236,18 +236,14 @@ impl Default for User { impl User { fn validate(&self) -> Result<(), Error> { - match self.min_pool_size { - Some(min_pool_size) => { - if min_pool_size > self.pool_size { - error!( - "min_pool_size of {} cannot be larger than pool_size of {}", - min_pool_size, self.pool_size - ); - return Err(Error::BadConfig); - } + if let Some(min_pool_size) = self.min_pool_size { + if min_pool_size > self.pool_size { + error!( + "min_pool_size of {} cannot be larger than pool_size of {}", + min_pool_size, self.pool_size + ); + return Err(Error::BadConfig); } - - None => (), }; Ok(()) @@ -677,9 +673,9 @@ impl Pool { Some(key) => { // No quotes in the key so we don't have to compare quoted // to unquoted idents. - let key = key.replace("\"", ""); + let key = key.replace('\"', ""); - if key.split(".").count() != 2 { + if key.split('.').count() != 2 { error!( "automatic_sharding_key '{}' must be fully qualified, e.g. t.{}`", key, key @@ -692,17 +688,14 @@ impl Pool { None => None, }; - match self.default_shard { - DefaultShard::Shard(shard_number) => { - if shard_number >= self.shards.len() { - error!("Invalid shard {:?}", shard_number); - return Err(Error::BadConfig); - } + if let DefaultShard::Shard(shard_number) = self.default_shard { + if shard_number >= self.shards.len() { + error!("Invalid shard {:?}", shard_number); + return Err(Error::BadConfig); } - _ => (), } - for (_, user) in &self.users { + for user in self.users.values() { user.validate()?; } @@ -777,8 +770,8 @@ impl<'de> serde::Deserialize<'de> for DefaultShard { D: Deserializer<'de>, { let s = String::deserialize(deserializer)?; - if s.starts_with("shard_") { - let shard = s[6..].parse::().map_err(serde::de::Error::custom)?; + if let Some(s) = s.strip_prefix("shard_") { + let shard = s.parse::().map_err(serde::de::Error::custom)?; return Ok(DefaultShard::Shard(shard)); } @@ -874,7 +867,7 @@ pub trait Plugin { impl std::fmt::Display for Plugins { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { fn is_enabled(arg: Option<&T>) -> bool { - if let Some(ref arg) = arg { + if let Some(arg) = arg { arg.is_enabled() } else { false @@ -955,6 +948,7 @@ pub struct Query { } impl Query { + #[allow(clippy::needless_range_loop)] pub fn substitute(&mut self, db: &str, user: &str) { for col in self.result.iter_mut() { for i in 0..col.len() { @@ -1079,8 +1073,8 @@ impl From<&Config> for std::collections::HashMap { ( format!("pools.{:?}.users", pool_name), pool.users - .iter() - .map(|(_username, user)| &user.username) + .values() + .map(|user| &user.username) .cloned() .collect::>() .join(", "), @@ -1165,13 +1159,9 @@ impl Config { Some(tls_certificate) => { info!("TLS certificate: {}", tls_certificate); - match self.general.tls_private_key.clone() { - Some(tls_private_key) => { - info!("TLS private key: {}", tls_private_key); - info!("TLS support is enabled"); - } - - None => (), + if let Some(tls_private_key) = self.general.tls_private_key.clone() { + info!("TLS private key: {}", tls_private_key); + info!("TLS support is enabled"); } } @@ -1206,8 +1196,8 @@ impl Config { pool_name, pool_config .users - .iter() - .map(|(_, user_cfg)| user_cfg.pool_size) + .values() + .map(|user_cfg| user_cfg.pool_size) .sum::() .to_string() ); @@ -1377,34 +1367,31 @@ impl Config { } // Validate TLS! - match self.general.tls_certificate.clone() { - Some(tls_certificate) => { - match load_certs(Path::new(&tls_certificate)) { - Ok(_) => { - // Cert is okay, but what about the private key? - match self.general.tls_private_key.clone() { - Some(tls_private_key) => match load_keys(Path::new(&tls_private_key)) { - Ok(_) => (), - Err(err) => { - error!("tls_private_key is incorrectly configured: {:?}", err); - return Err(Error::BadConfig); - } - }, - - None => { - error!("tls_certificate is set, but the tls_private_key is not"); + if let Some(tls_certificate) = self.general.tls_certificate.clone() { + match load_certs(Path::new(&tls_certificate)) { + Ok(_) => { + // Cert is okay, but what about the private key? + match self.general.tls_private_key.clone() { + Some(tls_private_key) => match load_keys(Path::new(&tls_private_key)) { + Ok(_) => (), + Err(err) => { + error!("tls_private_key is incorrectly configured: {:?}", err); return Err(Error::BadConfig); } - }; - } + }, - Err(err) => { - error!("tls_certificate is incorrectly configured: {:?}", err); - return Err(Error::BadConfig); - } + None => { + error!("tls_certificate is set, but the tls_private_key is not"); + return Err(Error::BadConfig); + } + }; + } + + Err(err) => { + error!("tls_certificate is incorrectly configured: {:?}", err); + return Err(Error::BadConfig); } } - None => (), }; for pool in self.pools.values_mut() { diff --git a/src/messages.rs b/src/messages.rs index 07fe93176..86036a921 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -163,12 +163,10 @@ where match stream.write_all(&startup).await { Ok(_) => Ok(()), - Err(err) => { - return Err(Error::SocketError(format!( - "Error writing startup to server socket - Error: {:?}", - err - ))) - } + Err(err) => Err(Error::SocketError(format!( + "Error writing startup to server socket - Error: {:?}", + err + ))), } } @@ -244,8 +242,8 @@ pub fn md5_hash_password(user: &str, password: &str, salt: &[u8]) -> Vec { let mut md5 = Md5::new(); // First pass - md5.update(&password.as_bytes()); - md5.update(&user.as_bytes()); + md5.update(password.as_bytes()); + md5.update(user.as_bytes()); let output = md5.finalize_reset(); @@ -281,7 +279,7 @@ where { let password = md5_hash_password(user, password, salt); - let mut message = BytesMut::with_capacity(password.len() as usize + 5); + let mut message = BytesMut::with_capacity(password.len() + 5); message.put_u8(b'p'); message.put_i32(password.len() as i32 + 4); @@ -295,7 +293,7 @@ where S: tokio::io::AsyncWrite + std::marker::Unpin, { let password = md5_hash_second_pass(hash, salt); - let mut message = BytesMut::with_capacity(password.len() as usize + 5); + let mut message = BytesMut::with_capacity(password.len() + 5); message.put_u8(b'p'); message.put_i32(password.len() as i32 + 4); @@ -516,7 +514,7 @@ pub fn data_row_nullable(row: &Vec>) -> BytesMut { data_row.put_i32(column.len() as i32); data_row.put_slice(column); } else { - data_row.put_i32(-1 as i32); + data_row.put_i32(-1_i32); } } @@ -571,12 +569,10 @@ where { match stream.write_all(&buf).await { Ok(_) => Ok(()), - Err(err) => { - return Err(Error::SocketError(format!( - "Error writing to socket - Error: {:?}", - err - ))) - } + Err(err) => Err(Error::SocketError(format!( + "Error writing to socket - Error: {:?}", + err + ))), } } @@ -587,12 +583,10 @@ where { match stream.write_all(buf).await { Ok(_) => Ok(()), - Err(err) => { - return Err(Error::SocketError(format!( - "Error writing to socket - Error: {:?}", - err - ))) - } + Err(err) => Err(Error::SocketError(format!( + "Error writing to socket - Error: {:?}", + err + ))), } } @@ -603,19 +597,15 @@ where match stream.write_all(buf).await { Ok(_) => match stream.flush().await { Ok(_) => Ok(()), - Err(err) => { - return Err(Error::SocketError(format!( - "Error flushing socket - Error: {:?}", - err - ))) - } - }, - Err(err) => { - return Err(Error::SocketError(format!( - "Error writing to socket - Error: {:?}", + Err(err) => Err(Error::SocketError(format!( + "Error flushing socket - Error: {:?}", err - ))) - } + ))), + }, + Err(err) => Err(Error::SocketError(format!( + "Error writing to socket - Error: {:?}", + err + ))), } } @@ -730,7 +720,7 @@ impl BytesMutReader for Cursor<&BytesMut> { let mut buf = vec![]; match self.read_until(b'\0', &mut buf) { Ok(_) => Ok(String::from_utf8_lossy(&buf[..buf.len() - 1]).to_string()), - Err(err) => return Err(Error::ParseBytesError(err.to_string())), + Err(err) => Err(Error::ParseBytesError(err.to_string())), } } } @@ -746,7 +736,7 @@ impl BytesMutReader for BytesMut { let string_bytes = self.split_to(index + 1); Ok(String::from_utf8_lossy(&string_bytes[..string_bytes.len() - 1]).to_string()) } - None => return Err(Error::ParseBytesError("Could not read string".to_string())), + None => Err(Error::ParseBytesError("Could not read string".to_string())), } } } @@ -1311,38 +1301,38 @@ mod tests { fn parse_fields() { let mut complete_msg = vec![]; let severity = "FATAL"; - complete_msg.extend(field('S', &severity)); - complete_msg.extend(field('V', &severity)); + complete_msg.extend(field('S', severity)); + complete_msg.extend(field('V', severity)); let error_code = "29P02"; - complete_msg.extend(field('C', &error_code)); + complete_msg.extend(field('C', error_code)); let message = "password authentication failed for user \"wrong_user\""; - complete_msg.extend(field('M', &message)); + complete_msg.extend(field('M', message)); let detail_msg = "super detailed message"; - complete_msg.extend(field('D', &detail_msg)); + complete_msg.extend(field('D', detail_msg)); let hint_msg = "hint detail here"; - complete_msg.extend(field('H', &hint_msg)); + complete_msg.extend(field('H', hint_msg)); complete_msg.extend(field('P', "123")); complete_msg.extend(field('p', "234")); let internal_query = "SELECT * from foo;"; - complete_msg.extend(field('q', &internal_query)); + complete_msg.extend(field('q', internal_query)); let where_msg = "where goes here"; - complete_msg.extend(field('W', &where_msg)); + complete_msg.extend(field('W', where_msg)); let schema_msg = "schema_name"; - complete_msg.extend(field('s', &schema_msg)); + complete_msg.extend(field('s', schema_msg)); let table_msg = "table_name"; - complete_msg.extend(field('t', &table_msg)); + complete_msg.extend(field('t', table_msg)); let column_msg = "column_name"; - complete_msg.extend(field('c', &column_msg)); + complete_msg.extend(field('c', column_msg)); let data_type_msg = "type_name"; - complete_msg.extend(field('d', &data_type_msg)); + complete_msg.extend(field('d', data_type_msg)); let constraint_msg = "constraint_name"; - complete_msg.extend(field('n', &constraint_msg)); + complete_msg.extend(field('n', constraint_msg)); let file_msg = "pgcat.c"; - complete_msg.extend(field('F', &file_msg)); + complete_msg.extend(field('F', file_msg)); complete_msg.extend(field('L', "335")); let routine_msg = "my_failing_routine"; - complete_msg.extend(field('R', &routine_msg)); + complete_msg.extend(field('R', routine_msg)); tracing_subscriber::fmt() .with_max_level(tracing::Level::INFO) @@ -1378,11 +1368,11 @@ mod tests { ); let mut only_mandatory_msg = vec![]; - only_mandatory_msg.extend(field('S', &severity)); - only_mandatory_msg.extend(field('V', &severity)); - only_mandatory_msg.extend(field('C', &error_code)); - only_mandatory_msg.extend(field('M', &message)); - only_mandatory_msg.extend(field('D', &detail_msg)); + only_mandatory_msg.extend(field('S', severity)); + only_mandatory_msg.extend(field('V', severity)); + only_mandatory_msg.extend(field('C', error_code)); + only_mandatory_msg.extend(field('M', message)); + only_mandatory_msg.extend(field('D', detail_msg)); let err_fields = PgErrorMsg::parse(only_mandatory_msg.clone()).unwrap(); info!("only mandatory fields: {}", &err_fields); diff --git a/src/mirrors.rs b/src/mirrors.rs index 90bcd3555..f704a8cd2 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -137,18 +137,18 @@ impl MirroringManager { bytes_rx, disconnect_rx: exit_rx, }; - exit_senders.push(exit_tx.clone()); - byte_senders.push(bytes_tx.clone()); + exit_senders.push(exit_tx); + byte_senders.push(bytes_tx); client.start(); }); Self { - byte_senders: byte_senders, + byte_senders, disconnect_senders: exit_senders, } } - pub fn send(self: &mut Self, bytes: &BytesMut) { + pub fn send(&mut self, bytes: &BytesMut) { // We want to avoid performing an allocation if we won't be able to send the message // There is a possibility of a race here where we check the capacity and then the channel is // closed or the capacity is reduced to 0, but mirroring is best effort anyway @@ -170,7 +170,7 @@ impl MirroringManager { }); } - pub fn disconnect(self: &mut Self) { + pub fn disconnect(&mut self) { self.disconnect_senders .iter_mut() .for_each(|sender| match sender.try_send(()) { diff --git a/src/plugins/intercept.rs b/src/plugins/intercept.rs index 166294bc2..d13ab0734 100644 --- a/src/plugins/intercept.rs +++ b/src/plugins/intercept.rs @@ -92,7 +92,7 @@ impl<'a> Plugin for Intercept<'a> { .map(|s| { let s = s.as_str().to_string(); - if s == "" { + if s.is_empty() { None } else { Some(s) diff --git a/src/plugins/mod.rs b/src/plugins/mod.rs index 5ef6009a5..f1076d06a 100644 --- a/src/plugins/mod.rs +++ b/src/plugins/mod.rs @@ -33,6 +33,7 @@ pub enum PluginOutput { #[async_trait] pub trait Plugin { // Run before the query is sent to the server. + #[allow(clippy::ptr_arg)] async fn run( &mut self, query_router: &QueryRouter, diff --git a/src/plugins/prewarmer.rs b/src/plugins/prewarmer.rs index a09bbe9dd..cd93db9ad 100644 --- a/src/plugins/prewarmer.rs +++ b/src/plugins/prewarmer.rs @@ -20,7 +20,7 @@ impl<'a> Prewarmer<'a> { self.server.address(), query ); - self.server.query(&query).await?; + self.server.query(query).await?; } Ok(()) diff --git a/src/plugins/table_access.rs b/src/plugins/table_access.rs index 79c1260e4..b8153b5a8 100644 --- a/src/plugins/table_access.rs +++ b/src/plugins/table_access.rs @@ -34,7 +34,7 @@ impl<'a> Plugin for TableAccess<'a> { visit_relations(ast, |relation| { let relation = relation.to_string(); - let parts = relation.split(".").collect::>(); + let parts = relation.split('.').collect::>(); let table_name = parts.last().unwrap(); if self.tables.contains(&table_name.to_string()) { diff --git a/src/pool.rs b/src/pool.rs index 02dab2731..736dc1ade 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -241,20 +241,17 @@ impl ConnectionPool { let old_pool_ref = get_pool(pool_name, &user.username); let identifier = PoolIdentifier::new(pool_name, &user.username); - match old_pool_ref { - Some(pool) => { - // If the pool hasn't changed, get existing reference and insert it into the new_pools. - // We replace all pools at the end, but if the reference is kept, the pool won't get re-created (bb8). - if pool.config_hash == new_pool_hash_value { - info!( - "[pool: {}][user: {}] has not changed", - pool_name, user.username - ); - new_pools.insert(identifier.clone(), pool.clone()); - continue; - } + if let Some(pool) = old_pool_ref { + // If the pool hasn't changed, get existing reference and insert it into the new_pools. + // We replace all pools at the end, but if the reference is kept, the pool won't get re-created (bb8). + if pool.config_hash == new_pool_hash_value { + info!( + "[pool: {}][user: {}] has not changed", + pool_name, user.username + ); + new_pools.insert(identifier.clone(), pool.clone()); + continue; } - None => (), } info!( @@ -399,7 +396,7 @@ impl ConnectionPool { }, }; - let reaper_rate = *vec![idle_timeout, server_lifetime, POOL_REAPER_RATE] + let reaper_rate = *[idle_timeout, server_lifetime, POOL_REAPER_RATE] .iter() .min() .unwrap(); @@ -489,7 +486,7 @@ impl ConnectionPool { .clone() .map(|regex| Regex::new(regex.as_str()).unwrap()), regex_search_limit: pool_config.regex_search_limit.unwrap_or(1000), - default_shard: pool_config.default_shard.clone(), + default_shard: pool_config.default_shard, auth_query: pool_config.auth_query.clone(), auth_query_user: pool_config.auth_query_user.clone(), auth_query_password: pool_config.auth_query_password.clone(), @@ -678,7 +675,7 @@ impl ConnectionPool { let mut force_healthcheck = false; if self.is_banned(address) { - if self.try_unban(&address).await { + if self.try_unban(address).await { force_healthcheck = true; } else { debug!("Address {:?} is banned", address); @@ -806,8 +803,8 @@ impl ConnectionPool { // Don't leave a bad connection in the pool. server.mark_bad(); - self.ban(&address, BanReason::FailedHealthCheck, Some(client_info)); - return false; + self.ban(address, BanReason::FailedHealthCheck, Some(client_info)); + false } /// Ban an address (i.e. replica). It no longer will serve @@ -931,10 +928,10 @@ impl ConnectionPool { let guard = self.banlist.read(); for banlist in guard.iter() { for (address, (reason, timestamp)) in banlist.iter() { - bans.push((address.clone(), (reason.clone(), timestamp.clone()))); + bans.push((address.clone(), (reason.clone(), *timestamp))); } } - return bans; + bans } /// Get the address from the host url @@ -992,7 +989,7 @@ impl ConnectionPool { } let busy = provisioned - idle; debug!("{:?} has {:?} busy connections", address, busy); - return busy; + busy } fn valid_shard_id(&self, shard: Option) -> bool { @@ -1031,6 +1028,7 @@ pub struct ServerPool { } impl ServerPool { + #[allow(clippy::too_many_arguments)] pub fn new( address: Address, user: User, @@ -1043,7 +1041,7 @@ impl ServerPool { ) -> ServerPool { ServerPool { address, - user: user.clone(), + user, database: database.to_string(), client_server_map, auth_hash, diff --git a/src/query_router.rs b/src/query_router.rs index 189f2dcc6..8b451dd3f 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -91,7 +91,7 @@ impl QueryRouter { /// One-time initialization of regexes /// that parse our custom SQL protocol. pub fn setup() -> bool { - let set = match RegexSet::new(&CUSTOM_SQL_REGEXES) { + let set = match RegexSet::new(CUSTOM_SQL_REGEXES) { Ok(rgx) => rgx, Err(err) => { error!("QueryRouter::setup Could not compile regex set: {:?}", err); @@ -132,7 +132,7 @@ impl QueryRouter { self.pool_settings = pool_settings; } - pub fn pool_settings<'a>(&'a self) -> &'a PoolSettings { + pub fn pool_settings(&self) -> &PoolSettings { &self.pool_settings } @@ -148,7 +148,7 @@ impl QueryRouter { // Check for any sharding regex matches in any queries if comment_shard_routing_enabled { - match code as char { + match code { // For Parse and Query messages peek to see if they specify a shard_id as a comment early in the statement 'P' | 'Q' => { // Check only the first block of bytes configured by the pool settings @@ -344,16 +344,13 @@ impl QueryRouter { let code = message_cursor.get_u8() as char; let len = message_cursor.get_i32() as usize; - match self.pool_settings.query_parser_max_length { - Some(max_length) => { - if len > max_length { - return Err(Error::QueryRouterParserError(format!( - "Query too long for parser: {} > {}", - len, max_length - ))); - } + if let Some(max_length) = self.pool_settings.query_parser_max_length { + if len > max_length { + return Err(Error::QueryRouterParserError(format!( + "Query too long for parser: {} > {}", + len, max_length + ))); } - None => (), }; let query = match code { @@ -467,22 +464,18 @@ impl QueryRouter { inferred_shard: Option, prev_inferred_shard: &mut Option, ) -> Result<(), Error> { - match inferred_shard { - Some(shard) => { - if let Some(prev_shard) = *prev_inferred_shard { - if prev_shard != shard { - debug!("Found more than one shard in the query, not supported yet"); - return Err(Error::QueryRouterParserError( - "multiple shards in query".into(), - )); - } + if let Some(shard) = inferred_shard { + if let Some(prev_shard) = *prev_inferred_shard { + if prev_shard != shard { + debug!("Found more than one shard in the query, not supported yet"); + return Err(Error::QueryRouterParserError( + "multiple shards in query".into(), + )); } - *prev_inferred_shard = Some(shard); - self.active_shard = Some(shard); - debug!("Automatically using shard: {:?}", self.active_shard); } - - None => (), + *prev_inferred_shard = Some(shard); + self.active_shard = Some(shard); + debug!("Automatically using shard: {:?}", self.active_shard); }; Ok(()) } @@ -513,7 +506,7 @@ impl QueryRouter { assert!(after_columns.is_empty()); Self::process_table(table_name, &mut table_names); - Self::process_query(&*source, &mut exprs, &mut table_names, &Some(columns)); + Self::process_query(source, &mut exprs, &mut table_names, &Some(columns)); } Delete { tables, @@ -529,7 +522,7 @@ impl QueryRouter { // Multi tables delete are not supported in postgres. assert!(tables.is_empty()); - Self::process_tables_with_join(&from, &mut exprs, &mut table_names); + Self::process_tables_with_join(from, &mut exprs, &mut table_names); if let Some(using_tbl_with_join) = using { Self::process_tables_with_join( using_tbl_with_join, @@ -569,7 +562,7 @@ impl QueryRouter { ) { match &*query.body { SetExpr::Query(query) => { - Self::process_query(&*query, exprs, table_names, columns); + Self::process_query(query, exprs, table_names, columns); } // SELECT * FROM ... @@ -611,7 +604,7 @@ impl QueryRouter { } fn process_tables_with_join( - tables: &Vec, + tables: &[TableWithJoins], exprs: &mut Vec, table_names: &mut Vec>, ) { @@ -625,37 +618,21 @@ impl QueryRouter { exprs: &mut Vec, table_names: &mut Vec>, ) { - match &table.relation { - TableFactor::Table { name, .. } => { - Self::process_table(name, table_names); - } - - _ => (), + if let TableFactor::Table { name, .. } = &table.relation { + Self::process_table(name, table_names); }; // Get table names from all the joins. for join in table.joins.iter() { - match &join.relation { - TableFactor::Table { name, .. } => { - Self::process_table(name, table_names); - } - - _ => (), + if let TableFactor::Table { name, .. } = &join.relation { + Self::process_table(name, table_names); }; // We can filter results based on join conditions, e.g. // SELECT * FROM t INNER JOIN B ON B.sharding_key = 5; - match &join.join_operator { - JoinOperator::Inner(inner_join) => match &inner_join { - JoinConstraint::On(expr) => { - // Parse the selection criteria later. - exprs.push(expr.clone()); - } - - _ => (), - }, - - _ => (), + if let JoinOperator::Inner(JoinConstraint::On(expr)) = &join.join_operator { + // Parse the selection criteria later. + exprs.push(expr.clone()); }; } } @@ -814,7 +791,7 @@ impl QueryRouter { .automatic_sharding_key .as_ref() .unwrap() - .split(".") + .split('.') .map(|ident| Ident::new(ident.to_lowercase())) .collect::>(); @@ -822,12 +799,12 @@ impl QueryRouter { assert_eq!(sharding_key.len(), 2); for a in assignments { - if sharding_key[0].value == "*" { - if sharding_key[1].value == a.id.last().unwrap().value.to_lowercase() { - return Err(Error::QueryRouterParserError( - "Sharding key cannot be updated.".into(), - )); - } + if sharding_key[0].value == "*" + && sharding_key[1].value == a.id.last().unwrap().value.to_lowercase() + { + return Err(Error::QueryRouterParserError( + "Sharding key cannot be updated.".into(), + )); } } Ok(()) @@ -844,7 +821,7 @@ impl QueryRouter { .automatic_sharding_key .as_ref() .unwrap() - .split(".") + .split('.') .map(|ident| Ident::new(ident.to_lowercase())) .collect::>(); @@ -861,7 +838,7 @@ impl QueryRouter { Expr::Identifier(ident) => { // Only if we're dealing with only one table // and there is no ambiguity - if &ident.value.to_lowercase() == &sharding_key[1].value { + if ident.value.to_lowercase() == sharding_key[1].value { // Sharding key is unique enough, don't worry about // table names. if &sharding_key[0].value == "*" { @@ -874,13 +851,13 @@ impl QueryRouter { // SELECT * FROM t WHERE sharding_key = 5 // Make sure the table name from the sharding key matches // the table name from the query. - found = &sharding_key[0].value == &table[0].value.to_lowercase(); + found = sharding_key[0].value == table[0].value.to_lowercase(); } else if table.len() == 2 { // Table name is fully qualified with the schema: e.g. // SELECT * FROM public.t WHERE sharding_key = 5 // Ignore the schema (TODO: at some point, we want schema support) // and use the table name only. - found = &sharding_key[0].value == &table[1].value.to_lowercase(); + found = sharding_key[0].value == table[1].value.to_lowercase(); } else { debug!("Got table name with more than two idents, which is not possible"); } @@ -893,8 +870,8 @@ impl QueryRouter { // it will exist or Postgres will throw an error. if idents.len() == 2 { found = (&sharding_key[0].value == "*" - || &sharding_key[0].value == &idents[0].value.to_lowercase()) - && &sharding_key[1].value == &idents[1].value.to_lowercase(); + || sharding_key[0].value == idents[0].value.to_lowercase()) + && sharding_key[1].value == idents[1].value.to_lowercase(); } // TODO: key can have schema as well, e.g. public.data.id (len == 3) } @@ -926,7 +903,7 @@ impl QueryRouter { } Expr::Value(Value::Placeholder(placeholder)) => { - match placeholder.replace("$", "").parse::() { + match placeholder.replace('$', "").parse::() { Ok(placeholder) => result.push(ShardingKey::Placeholder(placeholder)), Err(_) => { debug!( @@ -1020,16 +997,16 @@ impl QueryRouter { db: &self.pool_settings.db, }; - let _ = query_logger.run(&self, ast).await; + let _ = query_logger.run(self, ast).await; } if let Some(ref intercept) = plugins.intercept { let mut intercept = Intercept { enabled: intercept.enabled, - config: &intercept, + config: intercept, }; - let result = intercept.run(&self, ast).await; + let result = intercept.run(self, ast).await; if let Ok(PluginOutput::Intercept(output)) = result { return Ok(PluginOutput::Intercept(output)); @@ -1042,7 +1019,7 @@ impl QueryRouter { tables: &table_access.tables, }; - let result = table_access.run(&self, ast).await; + let result = table_access.run(self, ast).await; if let Ok(PluginOutput::Deny(error)) = result { return Ok(PluginOutput::Deny(error)); @@ -1078,7 +1055,7 @@ impl QueryRouter { /// Should we attempt to parse queries? pub fn query_parser_enabled(&self) -> bool { - let enabled = match self.query_parser_enabled { + match self.query_parser_enabled { None => { debug!( "Using pool settings, query_parser_enabled: {}", @@ -1094,9 +1071,7 @@ impl QueryRouter { ); value } - }; - - enabled + } } pub fn primary_reads_enabled(&self) -> bool { @@ -1107,6 +1082,12 @@ impl QueryRouter { } } +impl Default for QueryRouter { + fn default() -> Self { + Self::new() + } +} + #[cfg(test)] mod test { use super::*; @@ -1128,10 +1109,14 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); qr.pool_settings.query_parser_read_write_splitting = true; - assert!(qr.try_execute_command(&simple_query("SET SERVER ROLE TO 'auto'")) != None); + assert!(qr + .try_execute_command(&simple_query("SET SERVER ROLE TO 'auto'")) + .is_some()); assert!(qr.query_parser_enabled()); - assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO off")) != None); + assert!(qr + .try_execute_command(&simple_query("SET PRIMARY READS TO off")) + .is_some()); let queries = vec![ simple_query("SELECT * FROM items WHERE id = 5"), @@ -1173,7 +1158,9 @@ mod test { QueryRouter::setup(); let mut qr = QueryRouter::new(); let query = simple_query("SELECT * FROM items WHERE id = 5"); - assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO on")) != None); + assert!(qr + .try_execute_command(&simple_query("SET PRIMARY READS TO on")) + .is_some()); assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); assert_eq!(qr.role(), None); @@ -1186,7 +1173,9 @@ mod test { qr.pool_settings.query_parser_read_write_splitting = true; qr.try_execute_command(&simple_query("SET SERVER ROLE TO 'auto'")); - assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO off")) != None); + assert!(qr + .try_execute_command(&simple_query("SET PRIMARY READS TO off")) + .is_some()); let prepared_stmt = BytesMut::from( &b"WITH t AS (SELECT * FROM items WHERE name = $1) SELECT * FROM t WHERE id = $2\0"[..], @@ -1356,9 +1345,11 @@ mod test { qr.pool_settings.query_parser_read_write_splitting = true; let query = simple_query("SET SERVER ROLE TO 'auto'"); - assert!(qr.try_execute_command(&simple_query("SET PRIMARY READS TO off")) != None); + assert!(qr + .try_execute_command(&simple_query("SET PRIMARY READS TO off")) + .is_some()); - assert!(qr.try_execute_command(&query) != None); + assert!(qr.try_execute_command(&query).is_some()); assert!(qr.query_parser_enabled()); assert_eq!(qr.role(), None); @@ -1372,7 +1363,7 @@ mod test { assert!(qr.query_parser_enabled()); let query = simple_query("SET SERVER ROLE TO 'default'"); - assert!(qr.try_execute_command(&query) != None); + assert!(qr.try_execute_command(&query).is_some()); assert!(!qr.query_parser_enabled()); } @@ -1420,11 +1411,11 @@ mod test { assert!(!qr.primary_reads_enabled()); let q1 = simple_query("SET SERVER ROLE TO 'primary'"); - assert!(qr.try_execute_command(&q1) != None); + assert!(qr.try_execute_command(&q1).is_some()); assert_eq!(qr.active_role.unwrap(), Role::Primary); let q2 = simple_query("SET SERVER ROLE TO 'default'"); - assert!(qr.try_execute_command(&q2) != None); + assert!(qr.try_execute_command(&q2).is_some()); assert_eq!(qr.active_role.unwrap(), pool_settings.default_role); } @@ -1485,29 +1476,29 @@ mod test { }; let mut qr = QueryRouter::new(); - qr.update_pool_settings(pool_settings.clone()); + qr.update_pool_settings(pool_settings); // Shard should start out unset assert_eq!(qr.active_shard, None); // Don't panic when short query eg. ; is sent let q0 = simple_query(";"); - assert!(qr.try_execute_command(&q0) == None); + assert!(qr.try_execute_command(&q0).is_none()); assert_eq!(qr.active_shard, None); // Make sure setting it works let q1 = simple_query("/* shard_id: 1 */ select 1 from foo;"); - assert!(qr.try_execute_command(&q1) == None); + assert!(qr.try_execute_command(&q1).is_none()); assert_eq!(qr.active_shard, Some(1)); // And make sure changing it works let q2 = simple_query("/* shard_id: 0 */ select 1 from foo;"); - assert!(qr.try_execute_command(&q2) == None); + assert!(qr.try_execute_command(&q2).is_none()); assert_eq!(qr.active_shard, Some(0)); // Validate setting by shard with expected shard copied from sharding.rs tests let q2 = simple_query("/* sharding_key: 6 */ select 1 from foo;"); - assert!(qr.try_execute_command(&q2) == None); + assert!(qr.try_execute_command(&q2).is_none()); assert_eq!(qr.active_shard, Some(2)); } @@ -1863,10 +1854,11 @@ mod test { }; QueryRouter::setup(); - let mut pool_settings = PoolSettings::default(); - pool_settings.query_parser_enabled = true; - pool_settings.plugins = Some(plugins); - + let pool_settings = PoolSettings { + query_parser_enabled: true, + plugins: Some(plugins), + ..Default::default() + }; let mut qr = QueryRouter::new(); qr.update_pool_settings(pool_settings); diff --git a/src/scram.rs b/src/scram.rs index 3e5d84705..111dd5e11 100644 --- a/src/scram.rs +++ b/src/scram.rs @@ -79,12 +79,12 @@ impl ScramSha256 { let server_message = Message::parse(message)?; if !server_message.nonce.starts_with(&self.nonce) { - return Err(Error::ProtocolSyncError(format!("SCRAM"))); + return Err(Error::ProtocolSyncError("SCRAM".to_string())); } let salt = match general_purpose::STANDARD.decode(&server_message.salt) { Ok(salt) => salt, - Err(_) => return Err(Error::ProtocolSyncError(format!("SCRAM"))), + Err(_) => return Err(Error::ProtocolSyncError("SCRAM".to_string())), }; let salted_password = Self::hi( @@ -166,9 +166,9 @@ impl ScramSha256 { pub fn finish(&mut self, message: &BytesMut) -> Result<(), Error> { let final_message = FinalMessage::parse(message)?; - let verifier = match general_purpose::STANDARD.decode(&final_message.value) { + let verifier = match general_purpose::STANDARD.decode(final_message.value) { Ok(verifier) => verifier, - Err(_) => return Err(Error::ProtocolSyncError(format!("SCRAM"))), + Err(_) => return Err(Error::ProtocolSyncError("SCRAM".to_string())), }; let mut hmac = match Hmac::::new_from_slice(&self.salted_password) { @@ -230,14 +230,14 @@ impl Message { .collect::>(); if parts.len() != 3 { - return Err(Error::ProtocolSyncError(format!("SCRAM"))); + return Err(Error::ProtocolSyncError("SCRAM".to_string())); } let nonce = str::replace(&parts[0], "r=", ""); let salt = str::replace(&parts[1], "s=", ""); let iterations = match str::replace(&parts[2], "i=", "").parse::() { Ok(iterations) => iterations, - Err(_) => return Err(Error::ProtocolSyncError(format!("SCRAM"))), + Err(_) => return Err(Error::ProtocolSyncError("SCRAM".to_string())), }; Ok(Message { @@ -257,7 +257,7 @@ impl FinalMessage { /// Parse the server final validation message. pub fn parse(message: &BytesMut) -> Result { if !message.starts_with(b"v=") || message.len() < 4 { - return Err(Error::ProtocolSyncError(format!("SCRAM"))); + return Err(Error::ProtocolSyncError("SCRAM".to_string())); } Ok(FinalMessage { diff --git a/src/server.rs b/src/server.rs index 70c8270d6..5ccafeeb8 100644 --- a/src/server.rs +++ b/src/server.rs @@ -197,12 +197,8 @@ impl ServerParameters { key = "DateStyle".to_string(); }; - if TRACKED_PARAMETERS.contains(&key) { + if TRACKED_PARAMETERS.contains(&key) || startup { self.parameters.insert(key, value); - } else { - if startup { - self.parameters.insert(key, value); - } } } @@ -332,6 +328,7 @@ pub struct Server { impl Server { /// Pretend to be the Postgres client and connect to the server given host, port and credentials. /// Perform the authentication and return the server in a ready for query state. + #[allow(clippy::too_many_arguments)] pub async fn startup( address: &Address, user: &User, @@ -440,10 +437,7 @@ impl Server { // Something else? m => { - return Err(Error::SocketError(format!( - "Unknown message: {}", - m as char - ))); + return Err(Error::SocketError(format!("Unknown message: {}", { m }))); } } } else { @@ -461,6 +455,8 @@ impl Server { None => &user.username, }; + #[allow(clippy::match_as_ref)] + #[allow(clippy::manual_map)] let password = match user.server_password { Some(ref server_password) => Some(server_password), None => match user.password { @@ -473,14 +469,11 @@ impl Server { let mut process_id: i32 = 0; let mut secret_key: i32 = 0; - let server_identifier = ServerIdentifier::new(username, &database); + let server_identifier = ServerIdentifier::new(username, database); // We'll be handling multiple packets, but they will all be structured the same. // We'll loop here until this exchange is complete. - let mut scram: Option = match password { - Some(password) => Some(ScramSha256::new(password)), - None => None, - }; + let mut scram: Option = password.map(|password| ScramSha256::new(password)); let mut server_parameters = ServerParameters::new(); @@ -882,7 +875,7 @@ impl Server { self.mirror_send(messages); self.stats().data_sent(messages.len()); - match write_all_flush(&mut self.stream, &messages).await { + match write_all_flush(&mut self.stream, messages).await { Ok(_) => { // Successfully sent to server self.last_activity = SystemTime::now(); @@ -1359,16 +1352,14 @@ impl Server { } pub fn mirror_send(&mut self, bytes: &BytesMut) { - match self.mirror_manager.as_mut() { - Some(manager) => manager.send(bytes), - None => (), + if let Some(manager) = self.mirror_manager.as_mut() { + manager.send(bytes) } } pub fn mirror_disconnect(&mut self) { - match self.mirror_manager.as_mut() { - Some(manager) => manager.disconnect(), - None => (), + if let Some(manager) = self.mirror_manager.as_mut() { + manager.disconnect() } } @@ -1397,7 +1388,7 @@ impl Server { server.send(&simple_query(query)).await?; let mut message = server.recv(None).await?; - Ok(parse_query_message(&mut message).await?) + parse_query_message(&mut message).await } } diff --git a/src/sharding.rs b/src/sharding.rs index 18581dcf1..a7a9df130 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -64,7 +64,7 @@ impl Sharder { fn sha1(&self, key: i64) -> usize { let mut hasher = Sha1::new(); - hasher.update(&key.to_string().as_bytes()); + hasher.update(key.to_string().as_bytes()); let result = hasher.finalize(); @@ -202,10 +202,10 @@ mod test { #[test] fn test_sha1_hash() { let sharder = Sharder::new(12, ShardingFunction::Sha1); - let ids = vec![ + let ids = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ]; - let shards = vec![ + let shards = [ 4, 7, 8, 3, 6, 0, 0, 10, 3, 11, 1, 7, 4, 4, 11, 2, 5, 0, 8, 3, ]; diff --git a/src/stats/pool.rs b/src/stats/pool.rs index d3ac78e9f..46c746329 100644 --- a/src/stats/pool.rs +++ b/src/stats/pool.rs @@ -86,11 +86,11 @@ impl PoolStats { } } - return map; + map } pub fn generate_header() -> Vec<(&'static str, DataType)> { - return vec![ + vec![ ("database", DataType::Text), ("user", DataType::Text), ("pool_mode", DataType::Text), @@ -105,11 +105,11 @@ impl PoolStats { ("sv_login", DataType::Numeric), ("maxwait", DataType::Numeric), ("maxwait_us", DataType::Numeric), - ]; + ] } pub fn generate_row(&self) -> Vec { - return vec![ + vec![ self.identifier.db.clone(), self.identifier.user.clone(), self.mode.to_string(), @@ -124,7 +124,7 @@ impl PoolStats { self.sv_login.to_string(), (self.maxwait / 1_000_000).to_string(), (self.maxwait % 1_000_000).to_string(), - ]; + ] } } From e428c44b6583951c8ed019e98295341a73f9ae4e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Wed, 11 Oct 2023 10:13:23 -0700 Subject: [PATCH 476/551] Removed unnecessary `clippy` overrides. (#614) Removed unnecessary clippy overrides. --- src/server.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/server.rs b/src/server.rs index 5ccafeeb8..3394cda72 100644 --- a/src/server.rs +++ b/src/server.rs @@ -455,14 +455,9 @@ impl Server { None => &user.username, }; - #[allow(clippy::match_as_ref)] - #[allow(clippy::manual_map)] - let password = match user.server_password { - Some(ref server_password) => Some(server_password), - None => match user.password { - Some(ref password) => Some(password), - None => None, - }, + let password = match user.server_password.as_ref() { + Some(server_password) => Some(server_password), + None => user.password.as_ref(), }; startup(&mut stream, username, database).await?; From f3d57b150b214e42890150e2f582d8e01c4e6fbd Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Fri, 20 Oct 2023 02:49:05 -0400 Subject: [PATCH 477/551] Reduces the amount of time the get_pool operation takes (#625) * Reduces the amount of time the get_pool operation takes * trigger build * Fix admin --- .gitignore | 1 + src/client.rs | 29 +++++++++++++++++------------ src/pool.rs | 18 +++++++++--------- src/query_router.rs | 10 +++++----- 4 files changed, 32 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 0b4361648..40d5de1b9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ lcov.info dev/.bash_history dev/cache !dev/cache/.keepme +.venv \ No newline at end of file diff --git a/src/client.rs b/src/client.rs index 98a0669c5..c3fd747ce 100644 --- a/src/client.rs +++ b/src/client.rs @@ -542,7 +542,7 @@ where } // Authenticate normal user. else { - let mut pool = match get_pool(pool_name, username) { + let pool = match get_pool(pool_name, username) { Some(pool) => pool, None => { error_response( @@ -800,6 +800,18 @@ where &self.pool_name, ); + // Get a pool instance referenced by the most up-to-date + // pointer. This ensures we always read the latest config + // when starting a query. + let mut pool = if self.admin { + // Admin clients do not use pools. + ConnectionPool::default() + } else { + self.get_pool().await? + }; + + query_router.update_pool_settings(&pool.settings); + // Our custom protocol loop. // We expect the client to either start a transaction with regular queries // or issue commands for our sharding and server selection protocol. @@ -853,12 +865,6 @@ where continue; } - // Get a pool instance referenced by the most up-to-date - // pointer. This ensures we always read the latest config - // when starting a query. - let mut pool = self.get_pool().await?; - query_router.update_pool_settings(pool.settings.clone()); - let mut initial_parsed_ast = None; match message[0] as char { @@ -990,12 +996,11 @@ where }; // Check if the pool is paused and wait until it's resumed. - if pool.wait_paused().await { - // Refresh pool information, something might have changed. - pool = self.get_pool().await?; - } + pool.wait_paused().await; - query_router.update_pool_settings(pool.settings.clone()); + // Refresh pool information, something might have changed. + pool = self.get_pool().await?; + query_router.update_pool_settings(&pool.settings); let current_shard = query_router.shard(); diff --git a/src/pool.rs b/src/pool.rs index 736dc1ade..77394070a 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -190,11 +190,11 @@ impl Default for PoolSettings { #[derive(Clone, Debug, Default)] pub struct ConnectionPool { /// The pools handled internally by bb8. - databases: Vec>>, + databases: Arc>>>, /// The addresses (host, port, role) to handle /// failover and load balancing deterministically. - addresses: Vec>, + addresses: Arc>>, /// List of banned addresses (see above) /// that should not be queried. @@ -206,7 +206,7 @@ pub struct ConnectionPool { original_server_parameters: Arc>, /// Pool configuration. - pub settings: PoolSettings, + pub settings: Arc, /// If not validated, we need to double check the pool is available before allowing a client /// to use it. @@ -445,13 +445,13 @@ impl ConnectionPool { } let pool = ConnectionPool { - databases: shards, - addresses, + databases: Arc::new(shards), + addresses: Arc::new(addresses), banlist: Arc::new(RwLock::new(banlist)), config_hash: new_pool_hash_value, original_server_parameters: Arc::new(RwLock::new(ServerParameters::new())), auth_hash: pool_auth_hash, - settings: PoolSettings { + settings: Arc::new(PoolSettings { pool_mode: match user.pool_mode { Some(pool_mode) => pool_mode, None => pool_config.pool_mode, @@ -494,7 +494,7 @@ impl ConnectionPool { Some(ref plugins) => Some(plugins.clone()), None => config.plugins.clone(), }, - }, + }), validated: Arc::new(AtomicBool::new(false)), paused: Arc::new(AtomicBool::new(false)), paused_waiter: Arc::new(Notify::new()), @@ -504,7 +504,7 @@ impl ConnectionPool { // before setting it globally. // Do this async and somewhere else, we don't have to wait here. if config.general.validate_config { - let mut validate_pool = pool.clone(); + let validate_pool = pool.clone(); tokio::task::spawn(async move { let _ = validate_pool.validate().await; }); @@ -525,7 +525,7 @@ impl ConnectionPool { /// when they connect. /// This also warms up the pool for clients that connect when /// the pooler starts up. - pub async fn validate(&mut self) -> Result<(), Error> { + pub async fn validate(&self) -> Result<(), Error> { let mut futures = Vec::new(); let validated = Arc::clone(&self.validated); diff --git a/src/query_router.rs b/src/query_router.rs index 8b451dd3f..541883f22 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -128,8 +128,8 @@ impl QueryRouter { } /// Pool settings can change because of a config reload. - pub fn update_pool_settings(&mut self, pool_settings: PoolSettings) { - self.pool_settings = pool_settings; + pub fn update_pool_settings(&mut self, pool_settings: &PoolSettings) { + self.pool_settings = pool_settings.clone(); } pub fn pool_settings(&self) -> &PoolSettings { @@ -1403,7 +1403,7 @@ mod test { assert_eq!(qr.primary_reads_enabled, None); // Internal state must not be changed due to this, only defaults - qr.update_pool_settings(pool_settings.clone()); + qr.update_pool_settings(&pool_settings); assert_eq!(qr.active_role, None); assert_eq!(qr.active_shard, None); @@ -1476,7 +1476,7 @@ mod test { }; let mut qr = QueryRouter::new(); - qr.update_pool_settings(pool_settings); + qr.update_pool_settings(&pool_settings); // Shard should start out unset assert_eq!(qr.active_shard, None); @@ -1860,7 +1860,7 @@ mod test { ..Default::default() }; let mut qr = QueryRouter::new(); - qr.update_pool_settings(pool_settings); + qr.update_pool_settings(&pool_settings); let query = simple_query("SELECT * FROM pg_database"); let ast = qr.parse(&query).unwrap(); From d31c6ec5340f9f16a5e8aec881f687fadfe28aca Mon Sep 17 00:00:00 2001 From: Zain Kabani <77307340+zainkabani@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:11:57 -0400 Subject: [PATCH 478/551] Reimplement prepared statements with LRU cache and statement deduplication (#618) * Initial commit * Cleanup and add stats * Use an arc instead of full clones to store the parse packets * Use mutex instead * fmt * clippy * fmt * fix? * fix? * fmt * typo * Update docs * Refactor custom protocol * fmt * move custom protocol handling to before parsing * Support describe * Add LRU for server side statement cache * rename variable * Refactoring * Move docs * Fix test * fix * Update tests * trigger build * Add more tests * Reorder handling sync * Support when a named describe is sent along with Parse (go pgx) and expecting results * don't talk to client if not needed when client sends Parse * fmt :( * refactor tests * nit * Reduce hashing * Reducing work done to decode describe and parse messages * minor refactor * Merge branch 'main' into zain/reimplment-prepared-statements-with-global-lru-cache * Rewrite extended and prepared protocol message handling to better support mocking response packets and close * An attempt to better handle if there are DDL changes that might break cached plans with ideas about how to further improve it * fix * Minor stats fixed and cleanup * Cosmetic fixes (#64) * Cosmetic fixes * fix test * Change server drop for statement cache error to a `deallocate all` * Updated comments and added new idea for handling DDL changes impacting cached plans * fix test? * Revert test change * trigger build, flakey test * Avoid potential race conditions by changing get_or_insert to promote for pool LRU * remove ps enabled variable on the server in favor of using an option * Add close to the Extended Protocol buffer --------- --- CONFIG.md | 25 +- Cargo.lock | 31 ++ Cargo.toml | 1 + pgcat.toml | 10 +- src/admin.rs | 5 + src/client.rs | 789 ++++++++++++++++++++++-------------- src/config.rs | 82 ++-- src/messages.rs | 201 +++++++-- src/mirrors.rs | 6 +- src/pool.rs | 93 +++++ src/server.rs | 188 ++++----- src/stats/server.rs | 3 + tests/pgbench/simple.sql | 2 +- tests/ruby/prepared_spec.rb | 207 +++++++++- 14 files changed, 1131 insertions(+), 512 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index fc118cb47..7635bf6ba 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -259,22 +259,6 @@ Password to be used for connecting to servers to obtain the hash used for md5 au specified in `auth_query_user`. The connection will be established using the database configured in the pool. This parameter is inherited by every pool and can be redefined in pool configuration. -### prepared_statements -``` -path: general.prepared_statements -default: false -``` - -Whether to use prepared statements or not. - -### prepared_statements_cache_size -``` -path: general.prepared_statements_cache_size -default: 500 -``` - -Size of the prepared statements cache. - ### dns_cache_enabled ``` path: general.dns_cache_enabled @@ -324,6 +308,15 @@ If the client doesn't specify, PgCat routes traffic to this role by default. `replica` round-robin between replicas only without touching the primary, `primary` all queries go to the primary unless otherwise specified. +### prepared_statements_cache_size +``` +path: general.prepared_statements_cache_size +default: 0 +``` + +Size of the prepared statements cache. 0 means disabled. +TODO: update documentation + ### query_parser_enabled ``` path: pools..query_parser_enabled diff --git a/Cargo.lock b/Cargo.lock index 929f1a81e..eeaf106c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.0.2" @@ -26,6 +37,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -553,6 +570,10 @@ name = "hashbrown" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "heck" @@ -821,6 +842,15 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +[[package]] +name = "lru" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" +dependencies = [ + "hashbrown 0.14.0", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -1008,6 +1038,7 @@ dependencies = [ "itertools", "jemallocator", "log", + "lru", "md-5", "nix", "num_cpus", diff --git a/Cargo.toml b/Cargo.toml index 805a4c7a7..3e5f7069b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ itertools = "0.10" clap = { version = "4.3.1", features = ["derive", "env"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter", "std"]} +lru = "0.12.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/pgcat.toml b/pgcat.toml index 772a1365e..841649eea 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -60,12 +60,6 @@ tcp_keepalives_count = 5 # Number of seconds between keepalive packets. tcp_keepalives_interval = 5 -# Handle prepared statements. -prepared_statements = true - -# Prepared statements server cache size. -prepared_statements_cache_size = 500 - # Path to TLS Certificate file to use for TLS connections # tls_certificate = ".circleci/server.cert" # Path to TLS private key file to use for TLS connections @@ -156,6 +150,10 @@ load_balancing_mode = "random" # `primary` all queries go to the primary unless otherwise specified. default_role = "any" +# Prepared statements cache size. +# TODO: update documentation +prepared_statements_cache_size = 500 + # If Query Parser is enabled, we'll attempt to parse # every incoming query to determine if it's a read or a write. # If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, diff --git a/src/admin.rs b/src/admin.rs index f1b0c63fa..80baa3fba 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -744,6 +744,7 @@ where ("age_seconds", DataType::Numeric), ("prepare_cache_hit", DataType::Numeric), ("prepare_cache_miss", DataType::Numeric), + ("prepare_cache_eviction", DataType::Numeric), ("prepare_cache_size", DataType::Numeric), ]; @@ -776,6 +777,10 @@ where .prepared_miss_count .load(Ordering::Relaxed) .to_string(), + server + .prepared_eviction_count + .load(Ordering::Relaxed) + .to_string(), server .prepared_cache_size .load(Ordering::Relaxed) diff --git a/src/client.rs b/src/client.rs index c3fd747ce..31dcb4bd2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -4,7 +4,7 @@ use crate::pool::BanReason; use bytes::{Buf, BufMut, BytesMut}; use log::{debug, error, info, trace, warn}; use once_cell::sync::Lazy; -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::sync::{atomic::AtomicUsize, Arc}; use std::time::Instant; use tokio::io::{split, AsyncReadExt, BufReader, ReadHalf, WriteHalf}; @@ -14,9 +14,7 @@ use tokio::sync::mpsc::Sender; use crate::admin::{generate_server_parameters_for_admin, handle_admin}; use crate::auth_passthrough::refetch_auth_hash; -use crate::config::{ - get_config, get_idle_client_in_transaction_timeout, get_prepared_statements, Address, PoolMode, -}; +use crate::config::{get_config, get_idle_client_in_transaction_timeout, Address, PoolMode}; use crate::constants::*; use crate::messages::*; use crate::plugins::PluginOutput; @@ -53,6 +51,9 @@ pub struct Client { /// them to the backend. buffer: BytesMut, + /// Used to buffer response messages to the client + response_message_queue_buffer: BytesMut, + /// Address addr: std::net::SocketAddr, @@ -102,8 +103,14 @@ pub struct Client { /// Used to notify clients about an impending shutdown shutdown: Receiver<()>, - /// Prepared statements - prepared_statements: HashMap, + /// Whether prepared statements are enabled for this client + prepared_statements_enabled: bool, + + /// Mapping of client named prepared statement to rewritten parse messages + prepared_statements: HashMap, u64)>, + + /// Buffered extended protocol data + extended_protocol_data_buffer: VecDeque, } /// Client entrypoint. @@ -518,6 +525,8 @@ where } }; + let mut prepared_statements_enabled = false; + // Authenticate admin user. let (transaction_mode, mut server_parameters) = if admin { let config = get_config(); @@ -651,6 +660,8 @@ where } let transaction_mode = pool.settings.pool_mode == PoolMode::Transaction; + prepared_statements_enabled = + transaction_mode && pool.prepared_statement_cache.is_some(); // If the pool hasn't been validated yet, // connect to the servers and figure out what's what. @@ -682,7 +693,7 @@ where auth_ok(&mut write).await?; write_all(&mut write, (&server_parameters).into()).await?; backend_key_data(&mut write, process_id, secret_key).await?; - ready_for_query(&mut write).await?; + send_ready_for_query(&mut write).await?; trace!("Startup OK"); let stats = Arc::new(ClientStats::new( @@ -696,8 +707,9 @@ where Ok(Client { read: BufReader::new(read), write, - addr, buffer: BytesMut::with_capacity(8196), + response_message_queue_buffer: BytesMut::with_capacity(8196), + addr, cancel_mode: false, transaction_mode, process_id, @@ -708,12 +720,14 @@ where admin, last_address_id: None, last_server_stats: None, + connected_to_server: false, pool_name: pool_name.clone(), username: username.clone(), server_parameters, shutdown, - connected_to_server: false, + prepared_statements_enabled, prepared_statements: HashMap::new(), + extended_protocol_data_buffer: VecDeque::new(), }) } @@ -731,8 +745,9 @@ where Ok(Client { read: BufReader::new(read), write, - addr, buffer: BytesMut::with_capacity(8196), + response_message_queue_buffer: BytesMut::with_capacity(8196), + addr, cancel_mode: true, transaction_mode: false, process_id, @@ -743,12 +758,14 @@ where admin: false, last_address_id: None, last_server_stats: None, + connected_to_server: false, pool_name: String::from("undefined"), username: String::from("undefined"), server_parameters: ServerParameters::new(), shutdown, - connected_to_server: false, + prepared_statements_enabled: false, prepared_statements: HashMap::new(), + extended_protocol_data_buffer: VecDeque::new(), }) } @@ -790,10 +807,6 @@ where // Result returned by one of the plugins. let mut plugin_output = None; - // Prepared statement being executed - let mut prepared_statement = None; - let mut will_prepare = false; - let client_identifier = ClientIdentifier::new( self.server_parameters.get_application_name(), &self.username, @@ -821,16 +834,13 @@ where self.transaction_mode ); - // Should we rewrite prepared statements and bind messages? - let mut prepared_statements_enabled = get_prepared_statements(); - // Read a complete message from the client, which normally would be // either a `Q` (query) or `P` (prepare, extended protocol). // We can parse it here before grabbing a server from the pool, // in case the client is sending some custom protocol messages, e.g. // SET SHARDING KEY TO 'bigint'; - let mut message = tokio::select! { + let message = tokio::select! { _ = self.shutdown.recv() => { if !self.admin { error_response_terminal( @@ -865,35 +875,18 @@ where continue; } + // Handle all custom protocol commands, if any. + if self + .handle_custom_protocol(&mut query_router, &message, &pool) + .await? + { + continue; + } + let mut initial_parsed_ast = None; match message[0] as char { - // Buffer extended protocol messages even if we do not have - // a server connection yet. Hopefully, when we get the S message - // we'll be able to allocate a connection. Also, clients do not expect - // the server to respond to these messages so even if we were not able to - // allocate a connection, we wouldn't be able to send back an error message - // to the client so we buffer them and defer the decision to error out or not - // to when we get the S message - 'D' => { - if prepared_statements_enabled { - let name; - (name, message) = self.rewrite_describe(message).await?; - - if let Some(name) = name { - prepared_statement = Some(name); - } - } - - self.buffer.put(&message[..]); - continue; - } - - 'E' => { - self.buffer.put(&message[..]); - continue; - } - + // Query 'Q' => { if query_router.query_parser_enabled() { match query_router.parse(&message) { @@ -928,14 +921,15 @@ where } } + // Buffer extended protocol messages even if we do not have + // a server connection yet. Hopefully, when we get the S message + // we'll be able to allocate a connection. Also, clients do not expect + // the server to respond to these messages so even if we were not able to + // allocate a connection, we wouldn't be able to send back an error message + // to the client so we buffer them and defer the decision to error out or not + // to when we get the S message + // Parse 'P' => { - if prepared_statements_enabled { - (prepared_statement, message) = self.rewrite_parse(message)?; - will_prepare = true; - } - - self.buffer.put(&message[..]); - if query_router.query_parser_enabled() { match query_router.parse(&message) { Ok(ast) => { @@ -954,34 +948,41 @@ where }; } + self.buffer_parse(message, &pool)?; + continue; } + // Bind 'B' => { - if prepared_statements_enabled { - (prepared_statement, message) = self.rewrite_bind(message).await?; - } - - self.buffer.put(&message[..]); - if query_router.query_parser_enabled() { query_router.infer_shard_from_bind(&message); } + self.buffer_bind(message).await?; + + continue; + } + + // Describe + 'D' => { + self.buffer_describe(message).await?; + continue; + } + + 'E' => { + self.extended_protocol_data_buffer + .push_back(ExtendedProtocolData::create_new_execute(message)); continue; } // Close (F) 'C' => { - if prepared_statements_enabled { - let close: Close = (&message).try_into()?; + let close: Close = (&message).try_into()?; - if close.is_prepared_statement() && !close.anonymous() { - self.prepared_statements.remove(&close.name); - write_all_flush(&mut self.write, &close_complete()).await?; - continue; - } - } + self.extended_protocol_data_buffer + .push_back(ExtendedProtocolData::create_new_close(message, close)); + continue; } _ => (), @@ -989,7 +990,7 @@ where // Check on plugin results. if let Some(PluginOutput::Deny(error)) = plugin_output { - self.buffer.clear(); + self.reset_buffered_state(); error_response(&mut self.write, &error).await?; plugin_output = None; continue; @@ -1002,77 +1003,6 @@ where pool = self.get_pool().await?; query_router.update_pool_settings(&pool.settings); - let current_shard = query_router.shard(); - - // Handle all custom protocol commands, if any. - match query_router.try_execute_command(&message) { - // Normal query, not a custom command. - None => (), - - // SET SHARD TO - Some((Command::SetShard, _)) => { - match query_router.shard() { - None => (), - Some(selected_shard) => { - if selected_shard >= pool.shards() { - // Bad shard number, send error message to client. - query_router.set_shard(current_shard); - - error_response( - &mut self.write, - &format!( - "shard {} is not configured {}, staying on shard {:?} (shard numbers start at 0)", - selected_shard, - pool.shards(), - current_shard, - ), - ) - .await?; - } else { - custom_protocol_response_ok(&mut self.write, "SET SHARD").await?; - } - } - } - continue; - } - - // SET PRIMARY READS TO - Some((Command::SetPrimaryReads, _)) => { - custom_protocol_response_ok(&mut self.write, "SET PRIMARY READS").await?; - continue; - } - - // SET SHARDING KEY TO - Some((Command::SetShardingKey, _)) => { - custom_protocol_response_ok(&mut self.write, "SET SHARDING KEY").await?; - continue; - } - - // SET SERVER ROLE TO - Some((Command::SetServerRole, _)) => { - custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; - continue; - } - - // SHOW SERVER ROLE - Some((Command::ShowServerRole, value)) => { - show_response(&mut self.write, "server role", &value).await?; - continue; - } - - // SHOW SHARD - Some((Command::ShowShard, value)) => { - show_response(&mut self.write, "shard", &value).await?; - continue; - } - - // SHOW PRIMARY READS - Some((Command::ShowPrimaryReads, value)) => { - show_response(&mut self.write, "primary reads", &value).await?; - continue; - } - }; - debug!("Waiting for connection from pool"); if !self.admin { self.stats.waiting(); @@ -1096,7 +1026,7 @@ where if message[0] as char == 'S' { error!("Got Sync message but failed to get a connection from the pool"); - self.buffer.clear(); + self.reset_buffered_state(); } error_response( @@ -1162,58 +1092,7 @@ where // If the client is in session mode, no more custom protocol // commands will be accepted. loop { - // Only check if we should rewrite prepared statements - // in session mode. In transaction mode, we check at the beginning of - // each transaction. - if !self.transaction_mode { - prepared_statements_enabled = get_prepared_statements(); - } - - debug!("Prepared statement active: {:?}", prepared_statement); - - // We are processing a prepared statement. - if let Some(ref name) = prepared_statement { - debug!("Checking prepared statement is on server"); - // Get the prepared statement the server expects to see. - let statement = match self.prepared_statements.get(name) { - Some(statement) => { - debug!("Prepared statement `{}` found in cache", name); - statement - } - None => { - return Err(Error::ClientError(format!( - "prepared statement `{}` not found", - name - ))) - } - }; - - // Since it's already in the buffer, we don't need to prepare it on this server. - if will_prepare { - server.will_prepare(&statement.name); - will_prepare = false; - } else { - // The statement is not prepared on the server, so we need to prepare it. - if server.should_prepare(&statement.name) { - match server.prepare(statement).await { - Ok(_) => (), - Err(err) => { - pool.ban( - &address, - BanReason::MessageSendFailed, - Some(&self.stats), - ); - return Err(err); - } - } - } - } - - // Done processing the prepared statement. - prepared_statement = None; - } - - let mut message = match initial_message { + let message = match initial_message { None => { trace!("Waiting for message inside transaction or in session mode"); @@ -1344,21 +1223,12 @@ where self.stats.disconnect(); self.release(); - if prepared_statements_enabled { - server.maintain_cache().await?; - } - return Ok(()); } // Parse // The query with placeholders is here, e.g. `SELECT * FROM users WHERE email = $1 AND active = $2`. 'P' => { - if prepared_statements_enabled { - (prepared_statement, message) = self.rewrite_parse(message)?; - will_prepare = true; - } - if query_router.query_parser_enabled() { if let Ok(ast) = query_router.parse(&message) { if let Ok(output) = query_router.execute_plugins(&ast).await { @@ -1367,55 +1237,35 @@ where } } - self.buffer.put(&message[..]); + self.buffer_parse(message, &pool)?; } // Bind // The placeholder's replacements are here, e.g. 'user@email.com' and 'true' 'B' => { - if prepared_statements_enabled { - (prepared_statement, message) = self.rewrite_bind(message).await?; - } - - self.buffer.put(&message[..]); + self.buffer_bind(message).await?; } // Describe // Command a client can issue to describe a previously prepared named statement. 'D' => { - if prepared_statements_enabled { - let name; - (name, message) = self.rewrite_describe(message).await?; - - if let Some(name) = name { - prepared_statement = Some(name); - } - } + self.buffer_describe(message).await?; + } - self.buffer.put(&message[..]); + // Execute + // Execute a prepared statement prepared in `P` and bound in `B`. + 'E' => { + self.extended_protocol_data_buffer + .push_back(ExtendedProtocolData::create_new_execute(message)); } + // Close // Close the prepared statement. 'C' => { - if prepared_statements_enabled { - let close: Close = (&message).try_into()?; - - if close.is_prepared_statement() && !close.anonymous() { - if let Some(parse) = self.prepared_statements.get(&close.name) { - server.will_close(&parse.generated_name); - } else { - // A prepared statement slipped through? Not impossible, since we don't support PREPARE yet. - }; - } - } - - self.buffer.put(&message[..]); - } + let close: Close = (&message).try_into()?; - // Execute - // Execute a prepared statement prepared in `P` and bound in `B`. - 'E' => { - self.buffer.put(&message[..]); + self.extended_protocol_data_buffer + .push_back(ExtendedProtocolData::create_new_close(message, close)); } // Sync @@ -1427,47 +1277,182 @@ where Some(PluginOutput::Deny(error)) => { error_response(&mut self.write, &error).await?; plugin_output = None; - self.buffer.clear(); + self.reset_buffered_state(); continue; } Some(PluginOutput::Intercept(result)) => { write_all(&mut self.write, result).await?; plugin_output = None; - self.buffer.clear(); + self.reset_buffered_state(); continue; } _ => (), }; + // Prepared statements can arrive like this + // 1. Without named describe + // Client: Parse, with name, query and params + // Sync + // Server: ParseComplete + // ReadyForQuery + // 3. Without named describe + // Client: Parse, with name, query and params + // Describe, with no name + // Sync + // Server: ParseComplete + // ParameterDescription + // RowDescription + // ReadyForQuery + // 2. With named describe + // Client: Parse, with name, query and params + // Describe, with name + // Sync + // Server: ParseComplete + // ParameterDescription + // RowDescription + // ReadyForQuery + + // Iterate over our extended protocol data that we've buffered + while let Some(protocol_data) = + self.extended_protocol_data_buffer.pop_front() + { + match protocol_data { + ExtendedProtocolData::Parse { data, metadata } => { + let (parse, hash) = match metadata { + Some(metadata) => metadata, + None => { + let first_char_in_name = *data.get(5).unwrap_or(&0); + if first_char_in_name != 0 { + // This is a named prepared statement while prepared statements are disabled + // Server connection state will need to be cleared at checkin + server.mark_dirty(); + } + // Not a prepared statement + self.buffer.put(&data[..]); + continue; + } + }; + + // This is a prepared statement we already have on the checked out server + if server.has_prepared_statement(&parse.name) { + debug!( + "Prepared statement `{}` found in server cache", + parse.name + ); + + // We don't want to send the parse message to the server + // Instead queue up a parse complete message to send to the client + self.response_message_queue_buffer.put(parse_complete()); + } else { + debug!( + "Prepared statement `{}` not found in server cache", + parse.name + ); + + // TODO: Consider adding the close logic that this function can send for eviction to the client buffer instead + // In this case we don't want to send the parse message to the server since the client is sending it + self.register_parse_to_server_cache( + false, &hash, &parse, &pool, server, &address, + ) + .await?; + + // Add parse message to buffer + self.buffer.put(&data[..]); + } + } + ExtendedProtocolData::Bind { data, metadata } => { + // This is using a prepared statement + if let Some(client_given_name) = metadata { + self.ensure_prepared_statement_is_on_server( + client_given_name, + &pool, + server, + &address, + ) + .await?; + } + + self.buffer.put(&data[..]); + } + ExtendedProtocolData::Describe { data, metadata } => { + // This is using a prepared statement + if let Some(client_given_name) = metadata { + self.ensure_prepared_statement_is_on_server( + client_given_name, + &pool, + server, + &address, + ) + .await?; + } + + self.buffer.put(&data[..]); + } + ExtendedProtocolData::Execute { data } => { + self.buffer.put(&data[..]) + } + ExtendedProtocolData::Close { data, close } => { + // We don't send the close message to the server if prepared statements are enabled + // and it's a close with a prepared statement name provided + if self.prepared_statements_enabled + && close.is_prepared_statement() + && !close.anonymous() + { + self.prepared_statements.remove(&close.name); + + // Queue up a close complete message to send to the client + self.response_message_queue_buffer.put(close_complete()); + } else { + self.buffer.put(&data[..]); + } + } + } + } + + // Add the sync message self.buffer.put(&message[..]); - let first_message_code = (*self.buffer.first().unwrap_or(&0)) as char; - - // Almost certainly true - if first_message_code == 'P' && !prepared_statements_enabled { - // Message layout - // P followed by 32 int followed by null-terminated statement name - // So message code should be in offset 0 of the buffer, first character - // in prepared statement name would be index 5 - let first_char_in_name = *self.buffer.get(5).unwrap_or(&0); - if first_char_in_name != 0 { - // This is a named prepared statement - // Server connection state will need to be cleared at checkin - server.mark_dirty(); + let mut should_send_to_server = true; + + // If we have just a sync message left (maybe after omitting sending some messages to the server) no need to send it to the server + if *self.buffer.first().unwrap() == b'S' { + should_send_to_server = false; + // queue up a ready for query message to send to the client, respecting the transaction state of the server + self.response_message_queue_buffer + .put(ready_for_query(server.in_transaction())); + } + + // Send all queued messages to the client + // NOTE: it's possible we don't perfectly send things back in the same order as postgres would, + // however clients should be able to handle this + if !self.response_message_queue_buffer.is_empty() { + if let Err(err) = write_all_flush( + &mut self.write, + &self.response_message_queue_buffer, + ) + .await + { + // We might be in some kind of error/in between protocol state + server.mark_bad(); + return Err(err); } + + self.response_message_queue_buffer.clear(); } - self.send_and_receive_loop( - code, - None, - server, - &address, - &pool, - &self.stats.clone(), - ) - .await?; + if should_send_to_server { + self.send_and_receive_loop( + code, + None, + server, + &address, + &pool, + &self.stats.clone(), + ) + .await?; + } self.buffer.clear(); @@ -1549,10 +1534,6 @@ where server.checkin_cleanup().await?; - if prepared_statements_enabled { - server.maintain_cache().await?; - } - server.stats().idle(); self.connected_to_server = false; @@ -1586,68 +1567,237 @@ where } } - /// Rewrite Parse (F) message to set the prepared statement name to one we control. - /// Save it into the client cache. - fn rewrite_parse(&mut self, message: BytesMut) -> Result<(Option, BytesMut), Error> { - let parse: Parse = (&message).try_into()?; + /// Handles custom protocol messages + /// Returns true if the message is custom protocol message, false otherwise + /// Does not work with prepared statements, only simple and extended protocol without parameters + async fn handle_custom_protocol( + &mut self, + query_router: &mut QueryRouter, + message: &BytesMut, + pool: &ConnectionPool, + ) -> Result { + let current_shard = query_router.shard(); + + match query_router.try_execute_command(message) { + None => Ok(false), + + Some(custom) => { + match custom { + // SET SHARD TO + (Command::SetShard, _) => { + match query_router.shard() { + None => {} + Some(selected_shard) => { + if selected_shard >= pool.shards() { + // Bad shard number, send error message to client. + query_router.set_shard(current_shard); + + error_response( + &mut self.write, + &format!( + "shard {} is not configured {}, staying on shard {:?} (shard numbers start at 0)", + selected_shard, + pool.shards(), + current_shard, + ), + ) + .await?; + } else { + custom_protocol_response_ok(&mut self.write, "SET SHARD") + .await?; + } + } + } + } + + // SET PRIMARY READS TO + (Command::SetPrimaryReads, _) => { + custom_protocol_response_ok(&mut self.write, "SET PRIMARY READS").await?; + } + + // SET SHARDING KEY TO + (Command::SetShardingKey, _) => { + custom_protocol_response_ok(&mut self.write, "SET SHARDING KEY").await?; + } + + // SET SERVER ROLE TO + (Command::SetServerRole, _) => { + custom_protocol_response_ok(&mut self.write, "SET SERVER ROLE").await?; + } + + // SHOW SERVER ROLE + (Command::ShowServerRole, value) => { + show_response(&mut self.write, "server role", &value).await?; + } + + // SHOW SHARD + (Command::ShowShard, value) => { + show_response(&mut self.write, "shard", &value).await?; + } + + // SHOW PRIMARY READS + (Command::ShowPrimaryReads, value) => { + show_response(&mut self.write, "primary reads", &value).await?; + } + }; + + Ok(true) + } + } + } - let name = parse.name.clone(); + /// Makes sure the the checked out server has the prepared statement and sends it to the server if it doesn't + async fn ensure_prepared_statement_is_on_server( + &mut self, + client_name: String, + pool: &ConnectionPool, + server: &mut Server, + address: &Address, + ) -> Result<(), Error> { + match self.prepared_statements.get(&client_name) { + Some((parse, hash)) => { + debug!("Prepared statement `{}` found in cache", parse.name); + // In this case we want to send the parse message to the server + // since pgcat is initiating the prepared statement on this specific server + self.register_parse_to_server_cache(true, hash, parse, pool, server, address) + .await?; + } + + None => { + return Err(Error::ClientError(format!( + "prepared statement `{}` not found", + client_name + ))) + } + }; + + Ok(()) + } + + /// Register the parse to the server cache and send it to the server if requested (ie. requested by pgcat) + /// + /// Also updates the pool LRU that this parse was used recently + async fn register_parse_to_server_cache( + &self, + should_send_parse_to_server: bool, + hash: &u64, + parse: &Arc, + pool: &ConnectionPool, + server: &mut Server, + address: &Address, + ) -> Result<(), Error> { + // We want to promote this in the pool's LRU + pool.promote_prepared_statement_hash(hash); - // Don't rewrite anonymous prepared statements - if parse.anonymous() { - debug!("Anonymous prepared statement"); - return Ok((None, message)); + if let Err(err) = server + .register_prepared_statement(parse, should_send_parse_to_server) + .await + { + pool.ban(address, BanReason::MessageSendFailed, Some(&self.stats)); + return Err(err); } - let parse = parse.rename(); + Ok(()) + } + + /// Register and rewrite the parse statement to the clients statement cache + /// and also the pool's statement cache. Add it to extended protocol data. + fn buffer_parse(&mut self, message: BytesMut, pool: &ConnectionPool) -> Result<(), Error> { + // Avoid parsing if prepared statements not enabled + let client_given_name = match self.prepared_statements_enabled { + true => Parse::get_name(&message)?, + false => "".to_string(), + }; + + if client_given_name.is_empty() { + debug!("Anonymous parse message"); + self.extended_protocol_data_buffer + .push_back(ExtendedProtocolData::create_new_parse(message, None)); + return Ok(()); + } + + let parse: Parse = (&message).try_into()?; + + // Compute the hash of the parse statement + let hash = parse.get_hash(); + + // Add the statement to the cache or check if we already have it + let new_parse = match pool.register_parse_to_cache(hash, &parse) { + Some(parse) => parse, + None => { + return Err(Error::ClientError(format!( + "Could not store Prepared statement `{}`", + client_given_name + ))) + } + }; debug!( "Renamed prepared statement `{}` to `{}` and saved to cache", - name, parse.name + client_given_name, new_parse.name ); - self.prepared_statements.insert(name.clone(), parse.clone()); + self.prepared_statements + .insert(client_given_name, (new_parse.clone(), hash)); - Ok((Some(name), parse.try_into()?)) + self.extended_protocol_data_buffer + .push_back(ExtendedProtocolData::create_new_parse( + new_parse.as_ref().try_into()?, + Some((new_parse.clone(), hash)), + )); + + Ok(()) } /// Rewrite the Bind (F) message to use the prepared statement name /// saved in the client cache. - async fn rewrite_bind( - &mut self, - message: BytesMut, - ) -> Result<(Option, BytesMut), Error> { - let bind: Bind = (&message).try_into()?; - let name = bind.prepared_statement.clone(); + async fn buffer_bind(&mut self, message: BytesMut) -> Result<(), Error> { + // Avoid parsing if prepared statements not enabled + let client_given_name = match self.prepared_statements_enabled { + true => Bind::get_name(&message)?, + false => "".to_string(), + }; - if bind.anonymous() { + if client_given_name.is_empty() { debug!("Anonymous bind message"); - return Ok((None, message)); + self.extended_protocol_data_buffer + .push_back(ExtendedProtocolData::create_new_bind(message, None)); + return Ok(()); } - match self.prepared_statements.get(&name) { - Some(prepared_stmt) => { - let bind = bind.reassign(prepared_stmt); + match self.prepared_statements.get(&client_given_name) { + Some((rewritten_parse, _)) => { + let message = Bind::rename(message, &rewritten_parse.name)?; - debug!("Rewrote bind `{}` to `{}`", name, bind.prepared_statement); + debug!( + "Rewrote bind `{}` to `{}`", + client_given_name, rewritten_parse.name + ); - Ok((Some(name), bind.try_into()?)) + self.extended_protocol_data_buffer.push_back( + ExtendedProtocolData::create_new_bind(message, Some(client_given_name)), + ); + + Ok(()) } None => { - debug!("Got bind for unknown prepared statement {:?}", bind); + debug!( + "Got bind for unknown prepared statement {:?}", + client_given_name + ); error_response( &mut self.write, &format!( "prepared statement \"{}\" does not exist", - bind.prepared_statement + client_given_name ), ) .await?; Err(Error::ClientError(format!( "Prepared statement `{}` doesn't exist", - name + client_given_name ))) } } @@ -1655,38 +1805,68 @@ where /// Rewrite the Describe (F) message to use the prepared statement name /// saved in the client cache. - async fn rewrite_describe( - &mut self, - message: BytesMut, - ) -> Result<(Option, BytesMut), Error> { - let describe: Describe = (&message).try_into()?; - let name = describe.statement_name.clone(); + async fn buffer_describe(&mut self, message: BytesMut) -> Result<(), Error> { + // Avoid parsing if prepared statements not enabled + let describe: Describe = match self.prepared_statements_enabled { + true => (&message).try_into()?, + false => Describe::empty_new(), + }; if describe.anonymous() { - debug!("Anonymous describe"); - return Ok((None, message)); + debug!("Anonymous describe message"); + self.extended_protocol_data_buffer + .push_back(ExtendedProtocolData::create_new_describe(message, None)); + + return Ok(()); } - match self.prepared_statements.get(&name) { - Some(prepared_stmt) => { - let describe = describe.rename(&prepared_stmt.name); + let client_given_name = describe.statement_name.clone(); + + match self.prepared_statements.get(&client_given_name) { + Some((rewritten_parse, _)) => { + let describe = describe.rename(&rewritten_parse.name); debug!( "Rewrote describe `{}` to `{}`", - name, describe.statement_name + client_given_name, describe.statement_name ); - Ok((Some(name), describe.try_into()?)) + self.extended_protocol_data_buffer.push_back( + ExtendedProtocolData::create_new_describe( + describe.try_into()?, + Some(client_given_name), + ), + ); + + Ok(()) } None => { debug!("Got describe for unknown prepared statement {:?}", describe); - Ok((None, message)) + error_response( + &mut self.write, + &format!( + "prepared statement \"{}\" does not exist", + client_given_name + ), + ) + .await?; + + Err(Error::ClientError(format!( + "Prepared statement `{}` doesn't exist", + client_given_name + ))) } } } + fn reset_buffered_state(&mut self) { + self.buffer.clear(); + self.extended_protocol_data_buffer.clear(); + self.response_message_queue_buffer.clear(); + } + /// Release the server from the client: it can't cancel its queries anymore. pub fn release(&self) { let mut guard = self.client_server_map.lock(); @@ -1723,6 +1903,7 @@ where match write_all_flush(&mut self.write, &response).await { Ok(_) => (), Err(err) => { + // We might be in some kind of error/in between protocol state, better to just kill this server server.mark_bad(); return Err(err); } diff --git a/src/config.rs b/src/config.rs index f91e488e2..3d140b9b2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -116,10 +116,10 @@ impl Default for Address { host: String::from("127.0.0.1"), port: 5432, shard: 0, - address_index: 0, - replica_number: 0, database: String::from("database"), role: Role::Replica, + replica_number: 0, + address_index: 0, username: String::from("username"), pool_name: String::from("pool_name"), mirrors: Vec::new(), @@ -337,12 +337,6 @@ pub struct General { pub auth_query: Option, pub auth_query_user: Option, pub auth_query_password: Option, - - #[serde(default)] - pub prepared_statements: bool, - - #[serde(default = "General::default_prepared_statements_cache_size")] - pub prepared_statements_cache_size: usize, } impl General { @@ -424,10 +418,6 @@ impl General { pub fn default_server_round_robin() -> bool { true } - - pub fn default_prepared_statements_cache_size() -> usize { - 500 - } } impl Default for General { @@ -439,35 +429,33 @@ impl Default for General { prometheus_exporter_port: 9930, connect_timeout: General::default_connect_timeout(), idle_timeout: General::default_idle_timeout(), - shutdown_timeout: Self::default_shutdown_timeout(), - healthcheck_timeout: Self::default_healthcheck_timeout(), - healthcheck_delay: Self::default_healthcheck_delay(), - ban_time: Self::default_ban_time(), - worker_threads: Self::default_worker_threads(), - idle_client_in_transaction_timeout: Self::default_idle_client_in_transaction_timeout(), tcp_keepalives_idle: Self::default_tcp_keepalives_idle(), tcp_keepalives_count: Self::default_tcp_keepalives_count(), tcp_keepalives_interval: Self::default_tcp_keepalives_interval(), tcp_user_timeout: Self::default_tcp_user_timeout(), log_client_connections: false, log_client_disconnections: false, - autoreload: None, dns_cache_enabled: false, dns_max_ttl: Self::default_dns_max_ttl(), + shutdown_timeout: Self::default_shutdown_timeout(), + healthcheck_timeout: Self::default_healthcheck_timeout(), + healthcheck_delay: Self::default_healthcheck_delay(), + ban_time: Self::default_ban_time(), + idle_client_in_transaction_timeout: Self::default_idle_client_in_transaction_timeout(), + server_lifetime: Self::default_server_lifetime(), + server_round_robin: Self::default_server_round_robin(), + worker_threads: Self::default_worker_threads(), + autoreload: None, tls_certificate: None, tls_private_key: None, server_tls: false, verify_server_certificate: false, admin_username: String::from("admin"), admin_password: String::from("admin"), + validate_config: true, auth_query: None, auth_query_user: None, auth_query_password: None, - server_lifetime: Self::default_server_lifetime(), - server_round_robin: Self::default_server_round_robin(), - validate_config: true, - prepared_statements: false, - prepared_statements_cache_size: 500, } } } @@ -568,6 +556,9 @@ pub struct Pool { #[serde(default)] // False pub log_client_parameter_status_changes: bool, + #[serde(default = "Pool::default_prepared_statements_cache_size")] + pub prepared_statements_cache_size: usize, + pub plugins: Option, pub shards: BTreeMap, pub users: BTreeMap, @@ -617,6 +608,10 @@ impl Pool { true } + pub fn default_prepared_statements_cache_size() -> usize { + 0 + } + pub fn validate(&mut self) -> Result<(), Error> { match self.default_role.as_ref() { "any" => (), @@ -708,17 +703,16 @@ impl Default for Pool { Pool { pool_mode: Self::default_pool_mode(), load_balancing_mode: Self::default_load_balancing_mode(), - shards: BTreeMap::from([(String::from("1"), Shard::default())]), - users: BTreeMap::default(), default_role: String::from("any"), query_parser_enabled: false, query_parser_max_length: None, query_parser_read_write_splitting: false, primary_reads_enabled: false, - sharding_function: ShardingFunction::PgBigintHash, - automatic_sharding_key: None, connect_timeout: None, idle_timeout: None, + server_lifetime: None, + sharding_function: ShardingFunction::PgBigintHash, + automatic_sharding_key: None, sharding_key_regex: None, shard_id_regex: None, regex_search_limit: Some(1000), @@ -726,10 +720,12 @@ impl Default for Pool { auth_query: None, auth_query_user: None, auth_query_password: None, - server_lifetime: None, - plugins: None, cleanup_server_connections: true, log_client_parameter_status_changes: false, + prepared_statements_cache_size: Self::default_prepared_statements_cache_size(), + plugins: None, + shards: BTreeMap::from([(String::from("1"), Shard::default())]), + users: BTreeMap::default(), } } } @@ -841,13 +837,13 @@ impl Shard { impl Default for Shard { fn default() -> Shard { Shard { + database: String::from("postgres"), + mirrors: None, servers: vec![ServerConfig { host: String::from("localhost"), port: 5432, role: Role::Primary, }], - mirrors: None, - database: String::from("postgres"), } } } @@ -1018,8 +1014,8 @@ impl Default for Config { Config { path: Self::default_path(), general: General::default(), - pools: HashMap::default(), plugins: None, + pools: HashMap::default(), } } } @@ -1128,6 +1124,7 @@ impl From<&Config> for std::collections::HashMap { impl Config { /// Print current configuration. pub fn show(&self) { + info!("Config path: {}", self.path); info!("Ban time: {}s", self.general.ban_time); info!( "Idle client in transaction timeout: {}ms", @@ -1174,13 +1171,6 @@ impl Config { "Server TLS certificate verification: {}", self.general.verify_server_certificate ); - info!("Prepared statements: {}", self.general.prepared_statements); - if self.general.prepared_statements { - info!( - "Prepared statements server cache size: {}", - self.general.prepared_statements_cache_size - ); - } info!( "Plugins: {}", match self.plugins { @@ -1271,6 +1261,10 @@ impl Config { "[pool: {}] Log client parameter status changes: {}", pool_name, pool_config.log_client_parameter_status_changes ); + info!( + "[pool: {}] Prepared statements server cache size: {}", + pool_name, pool_config.prepared_statements_cache_size + ); info!( "[pool: {}] Plugins: {}", pool_name, @@ -1413,14 +1407,6 @@ pub fn get_idle_client_in_transaction_timeout() -> u64 { CONFIG.load().general.idle_client_in_transaction_timeout } -pub fn get_prepared_statements() -> bool { - CONFIG.load().general.prepared_statements -} - -pub fn get_prepared_statements_cache_size() -> usize { - CONFIG.load().general.prepared_statements_cache_size -} - /// Parse the configuration file located at the path. pub async fn parse(path: &str) -> Result<(), Error> { let mut contents = String::new(); diff --git a/src/messages.rs b/src/messages.rs index 86036a921..3a26f42ac 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -12,13 +12,16 @@ use crate::config::get_config; use crate::errors::Error; use crate::constants::MESSAGE_TERMINATOR; +use std::collections::hash_map::DefaultHasher; use std::collections::HashMap; use std::ffi::CString; use std::fmt::{Display, Formatter}; +use std::hash::{Hash, Hasher}; use std::io::{BufRead, Cursor}; use std::mem; use std::str::FromStr; use std::sync::atomic::Ordering; +use std::sync::Arc; use std::time::Duration; /// Postgres data type mappings @@ -114,19 +117,11 @@ pub fn simple_query(query: &str) -> BytesMut { } /// Tell the client we're ready for another query. -pub async fn ready_for_query(stream: &mut S) -> Result<(), Error> +pub async fn send_ready_for_query(stream: &mut S) -> Result<(), Error> where S: tokio::io::AsyncWrite + std::marker::Unpin, { - let mut bytes = BytesMut::with_capacity( - mem::size_of::() + mem::size_of::() + mem::size_of::(), - ); - - bytes.put_u8(b'Z'); - bytes.put_i32(5); - bytes.put_u8(b'I'); // Idle - - write_all(stream, bytes).await + write_all(stream, ready_for_query(false)).await } /// Send the startup packet the server. We're pretending we're a Pg client. @@ -320,7 +315,7 @@ where res.put_slice(&set_complete[..]); write_all_half(stream, &res).await?; - ready_for_query(stream).await + send_ready_for_query(stream).await } /// Send a custom error message to the client. @@ -331,7 +326,7 @@ where S: tokio::io::AsyncWrite + std::marker::Unpin, { error_response_terminal(stream, message).await?; - ready_for_query(stream).await + send_ready_for_query(stream).await } /// Send a custom error message to the client. @@ -432,7 +427,7 @@ where res.put(command_complete("SELECT 1")); write_all_half(stream, &res).await?; - ready_for_query(stream).await + send_ready_for_query(stream).await } pub fn row_description(columns: &Vec<(&str, DataType)>) -> BytesMut { @@ -562,6 +557,37 @@ pub fn flush() -> BytesMut { bytes } +pub fn sync() -> BytesMut { + let mut bytes = BytesMut::with_capacity(mem::size_of::() + mem::size_of::()); + bytes.put_u8(b'S'); + bytes.put_i32(4); + bytes +} + +pub fn parse_complete() -> BytesMut { + let mut bytes = BytesMut::with_capacity(mem::size_of::() + mem::size_of::()); + + bytes.put_u8(b'1'); + bytes.put_i32(4); + bytes +} + +pub fn ready_for_query(in_transaction: bool) -> BytesMut { + let mut bytes = BytesMut::with_capacity( + mem::size_of::() + mem::size_of::() + mem::size_of::(), + ); + + bytes.put_u8(b'Z'); + bytes.put_i32(5); + if in_transaction { + bytes.put_u8(b'T'); + } else { + bytes.put_u8(b'I'); + } + + bytes +} + /// Write all data in the buffer to the TcpStream. pub async fn write_all(stream: &mut S, buf: BytesMut) -> Result<(), Error> where @@ -740,6 +766,51 @@ impl BytesMutReader for BytesMut { } } } + +pub enum ExtendedProtocolData { + Parse { + data: BytesMut, + metadata: Option<(Arc, u64)>, + }, + Bind { + data: BytesMut, + metadata: Option, + }, + Describe { + data: BytesMut, + metadata: Option, + }, + Execute { + data: BytesMut, + }, + Close { + data: BytesMut, + close: Close, + }, +} + +impl ExtendedProtocolData { + pub fn create_new_parse(data: BytesMut, metadata: Option<(Arc, u64)>) -> Self { + Self::Parse { data, metadata } + } + + pub fn create_new_bind(data: BytesMut, metadata: Option) -> Self { + Self::Bind { data, metadata } + } + + pub fn create_new_describe(data: BytesMut, metadata: Option) -> Self { + Self::Describe { data, metadata } + } + + pub fn create_new_execute(data: BytesMut) -> Self { + Self::Execute { data } + } + + pub fn create_new_close(data: BytesMut, close: Close) -> Self { + Self::Close { data, close } + } +} + /// Parse (F) message. /// See: #[derive(Clone, Debug)] @@ -748,7 +819,6 @@ pub struct Parse { #[allow(dead_code)] len: i32, pub name: String, - pub generated_name: String, query: String, num_params: i16, param_types: Vec, @@ -774,7 +844,6 @@ impl TryFrom<&BytesMut> for Parse { code, len, name, - generated_name: prepared_statement_name(), query, num_params, param_types, @@ -823,11 +892,44 @@ impl TryFrom<&Parse> for BytesMut { } impl Parse { - pub fn rename(mut self) -> Self { - self.name = self.generated_name.to_string(); + /// Renames the prepared statement to a new name based on the global counter + pub fn rewrite(mut self) -> Self { + self.name = format!( + "PGCAT_{}", + PREPARED_STATEMENT_COUNTER.fetch_add(1, Ordering::SeqCst) + ); self } + /// Gets the name of the prepared statement from the buffer + pub fn get_name(buf: &BytesMut) -> Result { + let mut cursor = Cursor::new(buf); + // Skip the code and length + cursor.advance(mem::size_of::() + mem::size_of::()); + cursor.read_string() + } + + /// Hashes the parse statement to be used as a key in the global cache + pub fn get_hash(&self) -> u64 { + // TODO_ZAIN: Take a look at which hashing function is being used + let mut hasher = DefaultHasher::new(); + + let concatenated = format!( + "{}{}{}", + self.query, + self.num_params, + self.param_types + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + ); + + concatenated.hash(&mut hasher); + + hasher.finish() + } + pub fn anonymous(&self) -> bool { self.name.is_empty() } @@ -958,9 +1060,42 @@ impl TryFrom for BytesMut { } impl Bind { - pub fn reassign(mut self, parse: &Parse) -> Self { - self.prepared_statement = parse.name.clone(); - self + /// Gets the name of the prepared statement from the buffer + pub fn get_name(buf: &BytesMut) -> Result { + let mut cursor = Cursor::new(buf); + // Skip the code and length + cursor.advance(mem::size_of::() + mem::size_of::()); + cursor.read_string()?; + cursor.read_string() + } + + /// Renames the prepared statement to a new name + pub fn rename(buf: BytesMut, new_name: &str) -> Result { + let mut cursor = Cursor::new(&buf); + // Read basic data from the cursor + let code = cursor.get_u8(); + let current_len = cursor.get_i32(); + let portal = cursor.read_string()?; + let prepared_statement = cursor.read_string()?; + + // Calculate new length + let new_len = current_len + new_name.len() as i32 - prepared_statement.len() as i32; + + // Begin building the response buffer + let mut response_buf = BytesMut::with_capacity(new_len as usize + 1); + response_buf.put_u8(code); + response_buf.put_i32(new_len); + + // Put the portal and new name into the buffer + // Note: panic if the provided string contains null byte + response_buf.put_slice(CString::new(portal)?.as_bytes_with_nul()); + response_buf.put_slice(CString::new(new_name)?.as_bytes_with_nul()); + + // Add the remainder of the original buffer into the response + response_buf.put_slice(&buf[cursor.position() as usize..]); + + // Return the buffer + Ok(response_buf) } pub fn anonymous(&self) -> bool { @@ -1016,6 +1151,15 @@ impl TryFrom for BytesMut { } impl Describe { + pub fn empty_new() -> Describe { + Describe { + code: 'D', + len: 4 + 1 + 1, + target: 'S', + statement_name: "".to_string(), + } + } + pub fn rename(mut self, name: &str) -> Self { self.statement_name = name.to_string(); self @@ -1104,13 +1248,6 @@ pub fn close_complete() -> BytesMut { bytes } -pub fn prepared_statement_name() -> String { - format!( - "P_{}", - PREPARED_STATEMENT_COUNTER.fetch_add(1, Ordering::SeqCst) - ) -} - // from https://www.postgresql.org/docs/12/protocol-error-fields.html #[derive(Debug, Default, PartialEq)] pub struct PgErrorMsg { @@ -1193,7 +1330,7 @@ impl Display for PgErrorMsg { } impl PgErrorMsg { - pub fn parse(error_msg: Vec) -> Result { + pub fn parse(error_msg: &[u8]) -> Result { let mut out = PgErrorMsg { severity_localized: "".to_string(), severity: "".to_string(), @@ -1341,7 +1478,7 @@ mod tests { info!( "full message: {}", - PgErrorMsg::parse(complete_msg.clone()).unwrap() + PgErrorMsg::parse(&complete_msg).unwrap() ); assert_eq!( PgErrorMsg { @@ -1364,7 +1501,7 @@ mod tests { line: Some(335), routine: Some(routine_msg.to_string()), }, - PgErrorMsg::parse(complete_msg).unwrap() + PgErrorMsg::parse(&complete_msg).unwrap() ); let mut only_mandatory_msg = vec![]; @@ -1374,7 +1511,7 @@ mod tests { only_mandatory_msg.extend(field('M', message)); only_mandatory_msg.extend(field('D', detail_msg)); - let err_fields = PgErrorMsg::parse(only_mandatory_msg.clone()).unwrap(); + let err_fields = PgErrorMsg::parse(&only_mandatory_msg).unwrap(); info!("only mandatory fields: {}", &err_fields); error!( "server error: {}: {}", @@ -1401,7 +1538,7 @@ mod tests { line: None, routine: None, }, - PgErrorMsg::parse(only_mandatory_msg).unwrap() + PgErrorMsg::parse(&only_mandatory_msg).unwrap() ); } } diff --git a/src/mirrors.rs b/src/mirrors.rs index f704a8cd2..5c39504e7 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -23,14 +23,15 @@ impl MirroredClient { async fn create_pool(&self) -> Pool { let config = get_config(); let default = std::time::Duration::from_millis(10_000).as_millis() as u64; - let (connection_timeout, idle_timeout, _cfg) = + let (connection_timeout, idle_timeout, _cfg, prepared_statement_cache_size) = match config.pools.get(&self.address.pool_name) { Some(cfg) => ( cfg.connect_timeout.unwrap_or(default), cfg.idle_timeout.unwrap_or(default), cfg.clone(), + cfg.prepared_statements_cache_size, ), - None => (default, default, crate::config::Pool::default()), + None => (default, default, crate::config::Pool::default(), 0), }; let manager = ServerPool::new( @@ -42,6 +43,7 @@ impl MirroredClient { None, true, false, + prepared_statement_cache_size, ); Pool::builder() diff --git a/src/pool.rs b/src/pool.rs index 77394070a..0de16c2a2 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -3,6 +3,7 @@ use async_trait::async_trait; use bb8::{ManageConnection, Pool, PooledConnection, QueueStrategy}; use chrono::naive::NaiveDateTime; use log::{debug, error, info, warn}; +use lru::LruCache; use once_cell::sync::Lazy; use parking_lot::{Mutex, RwLock}; use rand::seq::SliceRandom; @@ -10,6 +11,7 @@ use rand::thread_rng; use regex::Regex; use std::collections::HashMap; use std::fmt::{Display, Formatter}; +use std::num::NonZeroUsize; use std::sync::atomic::AtomicU64; use std::sync::{ atomic::{AtomicBool, Ordering}, @@ -24,6 +26,7 @@ use crate::config::{ use crate::errors::Error; use crate::auth_passthrough::AuthPassthrough; +use crate::messages::Parse; use crate::plugins::prewarmer; use crate::server::{Server, ServerParameters}; use crate::sharding::ShardingFunction; @@ -54,6 +57,57 @@ pub enum BanReason { AdminBan(i64), } +pub type PreparedStatementCacheType = Arc>; + +// TODO: Add stats the this cache +// TODO: Add application name to the cache value to help identify which application is using the cache +// TODO: Create admin command to show which statements are in the cache +#[derive(Debug)] +pub struct PreparedStatementCache { + cache: LruCache>, +} + +impl PreparedStatementCache { + pub fn new(mut size: usize) -> Self { + // Cannot be zeros + if size == 0 { + size = 1; + } + + PreparedStatementCache { + cache: LruCache::new(NonZeroUsize::new(size).unwrap()), + } + } + + /// Adds the prepared statement to the cache if it doesn't exist with a new name + /// if it already exists will give you the existing parse + /// + /// Pass the hash to this so that we can do the compute before acquiring the lock + pub fn get_or_insert(&mut self, parse: &Parse, hash: u64) -> Arc { + match self.cache.get(&hash) { + Some(rewritten_parse) => rewritten_parse.clone(), + None => { + let new_parse = Arc::new(parse.clone().rewrite()); + let evicted = self.cache.push(hash, new_parse.clone()); + + if let Some((_, evicted_parse)) = evicted { + debug!( + "Evicted prepared statement {} from cache", + evicted_parse.name + ); + } + + new_parse + } + } + } + + /// Marks the hash as most recently used if it exists + pub fn promote(&mut self, hash: &u64) { + self.cache.promote(hash); + } +} + /// An identifier for a PgCat pool, /// a database visible to clients. #[derive(Hash, Debug, Clone, PartialEq, Eq, Default)] @@ -223,6 +277,9 @@ pub struct ConnectionPool { /// AuthInfo pub auth_hash: Arc>>, + + /// Cache + pub prepared_statement_cache: Option, } impl ConnectionPool { @@ -376,6 +433,7 @@ impl ConnectionPool { }, pool_config.cleanup_server_connections, pool_config.log_client_parameter_status_changes, + pool_config.prepared_statements_cache_size, ); let connect_timeout = match pool_config.connect_timeout { @@ -498,6 +556,12 @@ impl ConnectionPool { validated: Arc::new(AtomicBool::new(false)), paused: Arc::new(AtomicBool::new(false)), paused_waiter: Arc::new(Notify::new()), + prepared_statement_cache: match pool_config.prepared_statements_cache_size { + 0 => None, + _ => Some(Arc::new(Mutex::new(PreparedStatementCache::new( + pool_config.prepared_statements_cache_size, + )))), + }, }; // Connect to the servers to make sure pool configuration is valid @@ -998,6 +1062,29 @@ impl ConnectionPool { Some(shard) => shard < self.shards(), } } + + /// Register a parse statement to the pool's cache and return the rewritten parse + /// + /// Do not pass an anonymous parse statement to this function + pub fn register_parse_to_cache(&self, hash: u64, parse: &Parse) -> Option> { + // We should only be calling this function if the cache is enabled + match self.prepared_statement_cache { + Some(ref prepared_statement_cache) => { + let mut cache = prepared_statement_cache.lock(); + Some(cache.get_or_insert(parse, hash)) + } + None => None, + } + } + + /// Promote a prepared statement hash in the LRU + pub fn promote_prepared_statement_hash(&self, hash: &u64) { + // We should only be calling this function if the cache is enabled + if let Some(ref prepared_statement_cache) = self.prepared_statement_cache { + let mut cache = prepared_statement_cache.lock(); + cache.promote(hash); + } + } } /// Wrapper for the bb8 connection pool. @@ -1025,6 +1112,9 @@ pub struct ServerPool { /// Log client parameter status changes log_client_parameter_status_changes: bool, + + /// Prepared statement cache size + prepared_statement_cache_size: usize, } impl ServerPool { @@ -1038,6 +1128,7 @@ impl ServerPool { plugins: Option, cleanup_connections: bool, log_client_parameter_status_changes: bool, + prepared_statement_cache_size: usize, ) -> ServerPool { ServerPool { address, @@ -1048,6 +1139,7 @@ impl ServerPool { plugins, cleanup_connections, log_client_parameter_status_changes, + prepared_statement_cache_size, } } } @@ -1078,6 +1170,7 @@ impl ManageConnection for ServerPool { self.auth_hash.clone(), self.cleanup_connections, self.log_client_parameter_status_changes, + self.prepared_statement_cache_size, ) .await { diff --git a/src/server.rs b/src/server.rs index 3394cda72..dff6a7694 100644 --- a/src/server.rs +++ b/src/server.rs @@ -3,12 +3,14 @@ use bytes::{Buf, BufMut, BytesMut}; use fallible_iterator::FallibleIterator; use log::{debug, error, info, trace, warn}; +use lru::LruCache; use once_cell::sync::Lazy; use parking_lot::{Mutex, RwLock}; use postgres_protocol::message; -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{HashMap, HashSet}; use std::mem; use std::net::IpAddr; +use std::num::NonZeroUsize; use std::sync::Arc; use std::time::SystemTime; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, BufStream}; @@ -16,7 +18,7 @@ use tokio::net::TcpStream; use tokio_rustls::rustls::{OwnedTrustAnchor, RootCertStore}; use tokio_rustls::{client::TlsStream, TlsConnector}; -use crate::config::{get_config, get_prepared_statements_cache_size, Address, User}; +use crate::config::{get_config, Address, User}; use crate::constants::*; use crate::dns_cache::{AddrSet, CACHED_RESOLVER}; use crate::errors::{Error, ServerIdentifier}; @@ -322,7 +324,7 @@ pub struct Server { log_client_parameter_status_changes: bool, /// Prepared statements - prepared_statements: BTreeSet, + prepared_statement_cache: Option>, } impl Server { @@ -338,6 +340,7 @@ impl Server { auth_hash: Arc>>, cleanup_connections: bool, log_client_parameter_status_changes: bool, + prepared_statement_cache_size: usize, ) -> Result { let cached_resolver = CACHED_RESOLVER.load(); let mut addr_set: Option = None; @@ -713,7 +716,7 @@ impl Server { } }; - let fields = match PgErrorMsg::parse(error) { + let fields = match PgErrorMsg::parse(&error) { Ok(f) => f, Err(err) => { return Err(err); @@ -818,7 +821,12 @@ impl Server { }, cleanup_connections, log_client_parameter_status_changes, - prepared_statements: BTreeSet::new(), + prepared_statement_cache: match prepared_statement_cache_size { + 0 => None, + _ => Some(LruCache::new( + NonZeroUsize::new(prepared_statement_cache_size).unwrap(), + )), + }, }; return Ok(server); @@ -957,6 +965,20 @@ impl Server { if self.in_copy_mode { self.in_copy_mode = false; } + + if self.prepared_statement_cache.is_some() { + let error_message = PgErrorMsg::parse(&message)?; + if error_message.message == "cached plan must not change result type" { + warn!("Server {:?} changed schema, dropping connection to clean up prepared statements", self.address); + // This will still result in an error to the client, but this server connection will drop all cached prepared statements + // so that any new queries will be re-prepared + // TODO: Other ideas to solve errors when there are DDL changes after a statement has been prepared + // - Recreate entire connection pool to force recreation of all server connections + // - Clear the ConnectionPool's statement cache so that new statement names are generated + // - Implement a retry (re-prepare) so the client doesn't see an error + self.cleanup_state.needs_cleanup_prepare = true; + } + } } // CommandComplete @@ -1067,115 +1089,92 @@ impl Server { Ok(bytes) } - /// Add the prepared statement to being tracked by this server. - /// The client is processing data that will create a prepared statement on this server. - pub fn will_prepare(&mut self, name: &str) { - debug!("Will prepare `{}`", name); - - self.prepared_statements.insert(name.to_string()); - self.stats.prepared_cache_add(); - } - - /// Check if we should prepare a statement on the server. - pub fn should_prepare(&self, name: &str) -> bool { - let should_prepare = !self.prepared_statements.contains(name); - - debug!("Should prepare `{}`: {}", name, should_prepare); + // Determines if the server already has a prepared statement with the given name + // Increments the prepared statement cache hit counter + pub fn has_prepared_statement(&mut self, name: &str) -> bool { + let cache = match &mut self.prepared_statement_cache { + Some(cache) => cache, + None => return false, + }; - if should_prepare { - self.stats.prepared_cache_miss(); - } else { + let has_it = cache.get(name).is_some(); + if has_it { self.stats.prepared_cache_hit(); + } else { + self.stats.prepared_cache_miss(); } - should_prepare + has_it } - /// Create a prepared statement on the server. - pub async fn prepare(&mut self, parse: &Parse) -> Result<(), Error> { - debug!("Preparing `{}`", parse.name); - - let bytes: BytesMut = parse.try_into()?; - self.send(&bytes).await?; - self.send(&flush()).await?; - - // Read and discard ParseComplete (B) - match read_message(&mut self.stream).await { - Ok(_) => (), - Err(err) => { - self.bad = true; - return Err(err); - } - } + pub fn add_prepared_statement_to_cache(&mut self, name: &str) -> Option { + let cache = match &mut self.prepared_statement_cache { + Some(cache) => cache, + None => return None, + }; - self.prepared_statements.insert(parse.name.to_string()); self.stats.prepared_cache_add(); - debug!("Prepared `{}`", parse.name); - - Ok(()) - } - - /// Maintain adequate cache size on the server. - pub async fn maintain_cache(&mut self) -> Result<(), Error> { - debug!("Cache maintenance run"); - - let max_cache_size = get_prepared_statements_cache_size(); - let mut names = Vec::new(); - - while self.prepared_statements.len() >= max_cache_size { - // The prepared statmeents are alphanumerically sorted by the BTree. - // FIFO. - if let Some(name) = self.prepared_statements.pop_last() { - names.push(name); + // If we evict something, we need to close it on the server + if let Some((evicted_name, _)) = cache.push(name.to_string(), ()) { + if evicted_name != name { + debug!( + "Evicted prepared statement {} from cache, replaced with {}", + evicted_name, name + ); + return Some(evicted_name); } - } - - if !names.is_empty() { - self.deallocate(names).await?; - } + }; - Ok(()) + None } - /// Remove the prepared statement from being tracked by this server. - /// The client is processing data that will cause the server to close the prepared statement. - pub fn will_close(&mut self, name: &str) { - debug!("Will close `{}`", name); + pub fn remove_prepared_statement_from_cache(&mut self, name: &str) { + let cache = match &mut self.prepared_statement_cache { + Some(cache) => cache, + None => return, + }; - self.prepared_statements.remove(name); + self.stats.prepared_cache_remove(); + cache.pop(name); } - /// Close a prepared statement on the server. - pub async fn deallocate(&mut self, names: Vec) -> Result<(), Error> { - for name in &names { - debug!("Deallocating prepared statement `{}`", name); + pub async fn register_prepared_statement( + &mut self, + parse: &Parse, + should_send_parse_to_server: bool, + ) -> Result<(), Error> { + if !self.has_prepared_statement(&parse.name) { + let mut bytes = BytesMut::new(); - let close = Close::new(name); - let bytes: BytesMut = close.try_into()?; + if should_send_parse_to_server { + let parse_bytes: BytesMut = parse.try_into()?; + bytes.extend_from_slice(&parse_bytes); + } - self.send(&bytes).await?; - } + // If we evict something, we need to close it on the server + // We do this by adding it to the messages we're sending to the server before the sync + if let Some(evicted_name) = self.add_prepared_statement_to_cache(&parse.name) { + self.remove_prepared_statement_from_cache(&evicted_name); + let close_bytes: BytesMut = Close::new(&evicted_name).try_into()?; + bytes.extend_from_slice(&close_bytes); + }; - if !names.is_empty() { - self.send(&flush()).await?; - } + // If we have a parse or close we need to send to the server, send them and sync + if !bytes.is_empty() { + bytes.extend_from_slice(&sync()); - // Read and discard CloseComplete (3) - for name in &names { - match read_message(&mut self.stream).await { - Ok(_) => { - self.prepared_statements.remove(name); - self.stats.prepared_cache_remove(); - debug!("Closed `{}`", name); - } + self.send(&bytes).await?; - Err(err) => { - self.bad = true; - return Err(err); + loop { + self.recv(None).await?; + + if !self.is_data_available() { + break; + } } - }; - } + } + }; Ok(()) } @@ -1312,6 +1311,10 @@ impl Server { if self.cleanup_state.needs_cleanup_prepare { reset_string.push_str("DEALLOCATE ALL;"); + // Since we deallocated all prepared statements, we need to clear the cache + if let Some(cache) = &mut self.prepared_statement_cache { + cache.clear(); + } }; self.query(&reset_string).await?; @@ -1377,6 +1380,7 @@ impl Server { Arc::new(RwLock::new(None)), true, false, + 0, ) .await?; debug!("Connected!, sending query."); diff --git a/src/stats/server.rs b/src/stats/server.rs index 443c0b6aa..5d2559948 100644 --- a/src/stats/server.rs +++ b/src/stats/server.rs @@ -49,6 +49,7 @@ pub struct ServerStats { pub error_count: Arc, pub prepared_hit_count: Arc, pub prepared_miss_count: Arc, + pub prepared_eviction_count: Arc, pub prepared_cache_size: Arc, } @@ -68,6 +69,7 @@ impl Default for ServerStats { reporter: get_reporter(), prepared_hit_count: Arc::new(AtomicU64::new(0)), prepared_miss_count: Arc::new(AtomicU64::new(0)), + prepared_eviction_count: Arc::new(AtomicU64::new(0)), prepared_cache_size: Arc::new(AtomicU64::new(0)), } } @@ -221,6 +223,7 @@ impl ServerStats { } pub fn prepared_cache_remove(&self) { + self.prepared_eviction_count.fetch_add(1, Ordering::Relaxed); self.prepared_cache_size.fetch_sub(1, Ordering::Relaxed); } } diff --git a/tests/pgbench/simple.sql b/tests/pgbench/simple.sql index ad5e6139f..a8429d2b5 100644 --- a/tests/pgbench/simple.sql +++ b/tests/pgbench/simple.sql @@ -36,4 +36,4 @@ SELECT abalance FROM pgbench_accounts WHERE aid = :aid; SET SERVER ROLE TO 'replica'; -- Read load balancing -SELECT abalance FROM pgbench_accounts WHERE aid = :aid; +SELECT abalance FROM pgbench_accounts WHERE aid = :aid; \ No newline at end of file diff --git a/tests/ruby/prepared_spec.rb b/tests/ruby/prepared_spec.rb index 58a30006b..8a31243b8 100644 --- a/tests/ruby/prepared_spec.rb +++ b/tests/ruby/prepared_spec.rb @@ -1,29 +1,214 @@ require_relative 'spec_helper' describe 'Prepared statements' do - let(:processes) { Helpers::Pgcat.three_shard_setup('sharded_db', 5) } + let(:pool_size) { 5 } + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", pool_size) } + let(:prepared_statements_cache_size) { 100 } + let(:server_round_robin) { false } - context 'enabled' do - it 'will work over the same connection' do + before do + new_configs = processes.pgcat.current_config + new_configs["general"]["server_round_robin"] = server_round_robin + new_configs["pools"]["sharded_db"]["prepared_statements_cache_size"] = prepared_statements_cache_size + new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = pool_size + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + end + + context 'when trying prepared statements' do + it 'it allows unparameterized statements to succeed' do + conn1 = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + conn2 = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + + prepared_query = "SELECT 1" + + # prepare query on server 1 and client 1 + conn1.prepare('statement1', prepared_query) + conn1.exec_prepared('statement1') + + conn2.transaction do + # Claim server 1 with client 2 + conn2.exec("SELECT 2") + + # Client 1 now runs the prepared query, and it's automatically + # prepared on server 2 + conn1.prepare('statement2', prepared_query) + conn1.exec_prepared('statement2') + + # Client 2 now prepares the same query that was already + # prepared on server 1. And PgBouncer reuses that already + # prepared query for this different client. + conn2.prepare('statement3', prepared_query) + conn2.exec_prepared('statement3') + end + ensure + conn1.close if conn1 + conn2.close if conn2 + end + + it 'it allows parameterized statements to succeed' do + conn1 = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + conn2 = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + + prepared_query = "SELECT $1" + + # prepare query on server 1 and client 1 + conn1.prepare('statement1', prepared_query) + conn1.exec_prepared('statement1', [1]) + + conn2.transaction do + # Claim server 1 with client 2 + conn2.exec("SELECT 2") + + # Client 1 now runs the prepared query, and it's automatically + # prepared on server 2 + conn1.prepare('statement2', prepared_query) + conn1.exec_prepared('statement2', [1]) + + # Client 2 now prepares the same query that was already + # prepared on server 1. And PgBouncer reuses that already + # prepared query for this different client. + conn2.prepare('statement3', prepared_query) + conn2.exec_prepared('statement3', [1]) + end + ensure + conn1.close if conn1 + conn2.close if conn2 + + end + end + + context 'when trying large packets' do + it "works with large parse" do + conn1 = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + + long_string = "1" * 4096 * 10 + prepared_query = "SELECT '#{long_string}'" + + + # prepare query on server 1 and client 1 + conn1.prepare('statement1', prepared_query) + result = conn1.exec_prepared('statement1') + + # assert result matches long_string + expect(result.getvalue(0, 0)).to eq(long_string) + ensure + conn1.close if conn1 + end + + it "works with large bind" do + conn1 = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + + long_string = "1" * 4096 * 10 + prepared_query = "SELECT $1::text" + + # prepare query on server 1 and client 1 + conn1.prepare('statement1', prepared_query) + result = conn1.exec_prepared('statement1', [long_string]) + + # assert result matches long_string + expect(result.getvalue(0, 0)).to eq(long_string) + ensure + conn1.close if conn1 + end + end + + context 'when statement cache is smaller than set of unqiue statements' do + let(:prepared_statements_cache_size) { 1 } + let(:pool_size) { 1 } + + it "evicts all but 1 statement from the server cache" do + conn = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + + 5.times do |i| + prepared_query = "SELECT '#{i}'" + conn.prepare("statement#{i}", prepared_query) + result = conn.exec_prepared("statement#{i}") + expect(result.getvalue(0, 0)).to eq(i.to_s) + end + + # Check number of prepared statements (expected: 1) + n_statements = conn.exec("SELECT count(*) FROM pg_prepared_statements").getvalue(0, 0).to_i + expect(n_statements).to eq(1) + end + end + + context 'when statement cache is larger than set of unqiue statements' do + let(:pool_size) { 1 } + + it "does not evict any of the statements from the cache" do + # cache size 5 + conn = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + + 5.times do |i| + prepared_query = "SELECT '#{i}'" + conn.prepare("statement#{i}", prepared_query) + result = conn.exec_prepared("statement#{i}") + expect(result.getvalue(0, 0)).to eq(i.to_s) + end + + # Check number of prepared statements (expected: 1) + n_statements = conn.exec("SELECT count(*) FROM pg_prepared_statements").getvalue(0, 0).to_i + expect(n_statements).to eq(5) + end + end + + context 'when preparing the same query' do + let(:prepared_statements_cache_size) { 5 } + let(:pool_size) { 5 } + + it "reuses statement cache when there are different statement names on the same connection" do conn = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) 10.times do |i| statement_name = "statement_#{i}" conn.prepare(statement_name, 'SELECT $1::int') conn.exec_prepared(statement_name, [1]) - conn.describe_prepared(statement_name) end + + # Check number of prepared statements (expected: 1) + n_statements = conn.exec("SELECT count(*) FROM pg_prepared_statements").getvalue(0, 0).to_i + expect(n_statements).to eq(1) end - it 'will work with new connections' do - 10.times do + it "reuses statement cache when there are different statement names on different connections" do + 10.times do |i| conn = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) - - statement_name = 'statement1' - conn.prepare('statement1', 'SELECT $1::int') - conn.exec_prepared('statement1', [1]) - conn.describe_prepared('statement1') + statement_name = "statement_#{i}" + conn.prepare(statement_name, 'SELECT $1::int') + conn.exec_prepared(statement_name, [1]) end + + # Check number of prepared statements (expected: 1) + conn = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + n_statements = conn.exec("SELECT count(*) FROM pg_prepared_statements").getvalue(0, 0).to_i + expect(n_statements).to eq(1) + end + end + + context 'when reloading config' do + let(:pool_size) { 1 } + + it "test_reload_config" do + conn = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + + # prepare query + conn.prepare('statement1', 'SELECT 1') + conn.exec_prepared('statement1') + + # Reload config which triggers pool recreation + new_configs = processes.pgcat.current_config + new_configs["pools"]["sharded_db"]["prepared_statements_cache_size"] = prepared_statements_cache_size + 1 + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + + # check that we're starting with no prepared statements on the server + conn_check = PG.connect(processes.pgcat.connection_string('sharded_db', 'sharding_user')) + n_statements = conn_check.exec("SELECT count(*) FROM pg_prepared_statements").getvalue(0, 0).to_i + expect(n_statements).to eq(0) + + # still able to run prepared query + conn.exec_prepared('statement1') end end end From d6881f71c3bd26685999379cf332cee523fe9b63 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 26 Oct 2023 10:50:45 -0700 Subject: [PATCH 479/551] bump version (#629) --- .github/workflows/publish-deb-package.yml | 2 +- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-deb-package.yml b/.github/workflows/publish-deb-package.yml index bfd29f280..772945287 100644 --- a/.github/workflows/publish-deb-package.yml +++ b/.github/workflows/publish-deb-package.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "1.1.2-dev" + default: "1.1.2-dev1" jobs: build: strategy: diff --git a/Cargo.lock b/Cargo.lock index eeaf106c7..ff7fddf90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,7 +1020,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pgcat" -version = "1.1.2-dev" +version = "1.1.2-dev1" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 3e5f7069b..b79270eac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.1.2-dev" +version = "1.1.2-dev1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 1357268e111c53863fc2be6fa739dcbc45cd342c Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 6 Nov 2023 12:18:52 -0800 Subject: [PATCH 480/551] Add connet_timeout and idle_timeout to the user (#634) * Add connect_timeout to the user * Allow user to override connect timeout * version * lock * Add both timeouts to the user --- Cargo.lock | 2 +- Cargo.toml | 2 +- pgcat.toml | 2 ++ src/auth_passthrough.rs | 2 ++ src/config.rs | 22 ++++++++++++++++++++++ src/pool.rs | 14 ++++++++++---- 6 files changed, 38 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff7fddf90..e50439758 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,7 +1020,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pgcat" -version = "1.1.2-dev1" +version = "1.1.2-dev2" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index b79270eac..6485622a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.1.2-dev1" +version = "1.1.2-dev2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/pgcat.toml b/pgcat.toml index 841649eea..9e19c13b9 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -301,6 +301,8 @@ username = "other_user" password = "other_user" pool_size = 21 statement_timeout = 15000 +connect_timeout = 1000 +idle_timeout = 1000 # Shard configs are structured as pool..shards. # Each shard config contains a list of servers that make up the shard diff --git a/src/auth_passthrough.rs b/src/auth_passthrough.rs index fc0f6dc69..159847ed2 100644 --- a/src/auth_passthrough.rs +++ b/src/auth_passthrough.rs @@ -79,6 +79,8 @@ impl AuthPassthrough { pool_mode: None, server_lifetime: None, min_pool_size: None, + connect_timeout: None, + idle_timeout: None, }; let user = &address.username; diff --git a/src/config.rs b/src/config.rs index 3d140b9b2..ef7952f29 100644 --- a/src/config.rs +++ b/src/config.rs @@ -216,6 +216,8 @@ pub struct User { pub server_lifetime: Option, #[serde(default)] // 0 pub statement_timeout: u64, + pub connect_timeout: Option, + pub idle_timeout: Option, } impl Default for User { @@ -230,6 +232,8 @@ impl Default for User { statement_timeout: 0, pool_mode: None, server_lifetime: None, + connect_timeout: None, + idle_timeout: None, } } } @@ -1307,6 +1311,24 @@ impl Config { None => "default".to_string(), } ); + info!( + "[pool: {}][user: {}] Connection timeout: {}", + pool_name, + user.1.username, + match user.1.connect_timeout { + Some(connect_timeout) => format!("{}ms", connect_timeout), + None => "not set".to_string(), + } + ); + info!( + "[pool: {}][user: {}] Idle timeout: {}", + pool_name, + user.1.username, + match user.1.idle_timeout { + Some(idle_timeout) => format!("{}ms", idle_timeout), + None => "not set".to_string(), + } + ); } } } diff --git a/src/pool.rs b/src/pool.rs index 0de16c2a2..751f18759 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -436,14 +436,20 @@ impl ConnectionPool { pool_config.prepared_statements_cache_size, ); - let connect_timeout = match pool_config.connect_timeout { + let connect_timeout = match user.connect_timeout { Some(connect_timeout) => connect_timeout, - None => config.general.connect_timeout, + None => match pool_config.connect_timeout { + Some(connect_timeout) => connect_timeout, + None => config.general.connect_timeout, + }, }; - let idle_timeout = match pool_config.idle_timeout { + let idle_timeout = match user.idle_timeout { Some(idle_timeout) => idle_timeout, - None => config.general.idle_timeout, + None => match pool_config.idle_timeout { + Some(idle_timeout) => idle_timeout, + None => config.general.idle_timeout, + }, }; let server_lifetime = match user.server_lifetime { From 8b37c1cb24216cbefee3299789168f92307a445b Mon Sep 17 00:00:00 2001 From: Jakob Schultz-Falk Date: Wed, 8 Nov 2023 17:25:49 +0100 Subject: [PATCH 481/551] Update Dockerfile.ci with golang (#637) --- Dockerfile.ci | 5 ++++- tests/docker/Dockerfile | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index a750612c7..57d28b87e 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -9,6 +9,9 @@ RUN sudo apt-get update && \ sudo apt-get upgrade curl && \ cargo install cargo-binutils rustfilt && \ rustup component add llvm-tools-preview && \ - pip3 install psycopg2 && sudo gem install bundler && \ + pip3 install psycopg2 && sudo gem install bundler && \ wget -O /tmp/toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ sudo dpkg -i /tmp/toxiproxy-2.4.0.deb +RUN wget -O /tmp/go1.21.3.linux-$(dpkg --print-architecture).tar.gz https://go.dev/dl/go1.21.3.linux-$(dpkg --print-architecture).tar.gz && \ + sudo tar -C /usr/local -xzf /tmp/go1.21.3.linux-$(dpkg --print-architecture).tar.gz && \ + rm /tmp/go1.21.3.linux-$(dpkg --print-architecture).tar.gz diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 261adb05c..6eccddd17 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -8,3 +8,6 @@ RUN rustup component add llvm-tools-preview RUN sudo gem install bundler RUN wget -O toxiproxy-2.4.0.deb https://github.com/Shopify/toxiproxy/releases/download/v2.4.0/toxiproxy_2.4.0_linux_$(dpkg --print-architecture).deb && \ sudo dpkg -i toxiproxy-2.4.0.deb +RUN wget -O go1.21.3.linux-$(dpkg --print-architecture).tar.gz https://go.dev/dl/go1.21.3.linux-$(dpkg --print-architecture).tar.gz && \ + sudo tar -C /usr/local -xzf go1.21.3.linux-$(dpkg --print-architecture).tar.gz && \ + rm go1.21.3.linux-$(dpkg --print-architecture).tar.gz From c3213bc23ac7a547e671a4a9b3bd34d15cc4718e Mon Sep 17 00:00:00 2001 From: Jakob Schultz-Falk Date: Thu, 9 Nov 2023 01:36:45 +0100 Subject: [PATCH 482/551] Support unnamed prepared statements (#635) * Add golang test suite to reproduce issue with unnamed parameterized prepared statements * Allow caching of unnamed prepared statements * Passthrough describe on portals * Remove unneeded kill * Update Dockerfile.ci with golang * Move out update of Dockerfiles to separate PR --- .circleci/run_tests.sh | 9 +++ src/client.rs | 33 ++++---- src/messages.rs | 2 +- tests/go/go.mod | 5 ++ tests/go/go.sum | 2 + tests/go/pgcat.toml | 162 ++++++++++++++++++++++++++++++++++++++ tests/go/prepared_test.go | 52 ++++++++++++ tests/go/setup.go | 81 +++++++++++++++++++ 8 files changed, 327 insertions(+), 19 deletions(-) create mode 100644 tests/go/go.mod create mode 100644 tests/go/go.sum create mode 100644 tests/go/pgcat.toml create mode 100644 tests/go/prepared_test.go create mode 100644 tests/go/setup.go diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 4ba497c31..3a31240af 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -108,6 +108,15 @@ cd ../.. pip3 install -r tests/python/requirements.txt python3 tests/python/tests.py || exit 1 + +# +# Go tests +# Starts its own pgcat server +# +pushd tests/go +/usr/local/go/bin/go test || exit 1 +popd + start_pgcat "info" # Admin tests diff --git a/src/client.rs b/src/client.rs index 31dcb4bd2..dd89697c1 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1704,18 +1704,14 @@ where /// and also the pool's statement cache. Add it to extended protocol data. fn buffer_parse(&mut self, message: BytesMut, pool: &ConnectionPool) -> Result<(), Error> { // Avoid parsing if prepared statements not enabled - let client_given_name = match self.prepared_statements_enabled { - true => Parse::get_name(&message)?, - false => "".to_string(), - }; - - if client_given_name.is_empty() { + if !self.prepared_statements_enabled { debug!("Anonymous parse message"); self.extended_protocol_data_buffer .push_back(ExtendedProtocolData::create_new_parse(message, None)); return Ok(()); } + let client_given_name = Parse::get_name(&message)?; let parse: Parse = (&message).try_into()?; // Compute the hash of the parse statement @@ -1753,18 +1749,15 @@ where /// saved in the client cache. async fn buffer_bind(&mut self, message: BytesMut) -> Result<(), Error> { // Avoid parsing if prepared statements not enabled - let client_given_name = match self.prepared_statements_enabled { - true => Bind::get_name(&message)?, - false => "".to_string(), - }; - - if client_given_name.is_empty() { + if !self.prepared_statements_enabled { debug!("Anonymous bind message"); self.extended_protocol_data_buffer .push_back(ExtendedProtocolData::create_new_bind(message, None)); return Ok(()); } + let client_given_name = Bind::get_name(&message)?; + match self.prepared_statements.get(&client_given_name) { Some((rewritten_parse, _)) => { let message = Bind::rename(message, &rewritten_parse.name)?; @@ -1807,12 +1800,7 @@ where /// saved in the client cache. async fn buffer_describe(&mut self, message: BytesMut) -> Result<(), Error> { // Avoid parsing if prepared statements not enabled - let describe: Describe = match self.prepared_statements_enabled { - true => (&message).try_into()?, - false => Describe::empty_new(), - }; - - if describe.anonymous() { + if !self.prepared_statements_enabled { debug!("Anonymous describe message"); self.extended_protocol_data_buffer .push_back(ExtendedProtocolData::create_new_describe(message, None)); @@ -1820,6 +1808,15 @@ where return Ok(()); } + let describe: Describe = (&message).try_into()?; + if describe.target == 'P' { + debug!("Portal describe message"); + self.extended_protocol_data_buffer + .push_back(ExtendedProtocolData::create_new_describe(message, None)); + + return Ok(()); + } + let client_given_name = describe.statement_name.clone(); match self.prepared_statements.get(&client_given_name) { diff --git a/src/messages.rs b/src/messages.rs index 3a26f42ac..4390d9f9b 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -1109,7 +1109,7 @@ pub struct Describe { #[allow(dead_code)] len: i32, - target: char, + pub target: char, pub statement_name: String, } diff --git a/tests/go/go.mod b/tests/go/go.mod new file mode 100644 index 000000000..faa2292e6 --- /dev/null +++ b/tests/go/go.mod @@ -0,0 +1,5 @@ +module pgcat + +go 1.21 + +require github.com/lib/pq v1.10.9 diff --git a/tests/go/go.sum b/tests/go/go.sum new file mode 100644 index 000000000..aeddeae36 --- /dev/null +++ b/tests/go/go.sum @@ -0,0 +1,2 @@ +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= diff --git a/tests/go/pgcat.toml b/tests/go/pgcat.toml new file mode 100644 index 000000000..72eba8a4f --- /dev/null +++ b/tests/go/pgcat.toml @@ -0,0 +1,162 @@ +# +# PgCat config example. +# + +# +# General pooler settings +[general] +# What IP to run on, 0.0.0.0 means accessible from everywhere. +host = "0.0.0.0" + +# Port to run on, same as PgBouncer used in this example. +port = "${PORT}" + +# Whether to enable prometheus exporter or not. +enable_prometheus_exporter = true + +# Port at which prometheus exporter listens on. +prometheus_exporter_port = 9930 + +# How long to wait before aborting a server connection (ms). +connect_timeout = 1000 + +# How much time to give the health check query to return with a result (ms). +healthcheck_timeout = 1000 + +# How long to keep connection available for immediate re-use, without running a healthcheck query on it +healthcheck_delay = 30000 + +# How much time to give clients during shutdown before forcibly killing client connections (ms). +shutdown_timeout = 5000 + +# For how long to ban a server if it fails a health check (seconds). +ban_time = 60 # Seconds + +# If we should log client connections +log_client_connections = false + +# If we should log client disconnections +log_client_disconnections = false + +# Reload config automatically if it changes. +autoreload = 15000 + +server_round_robin = false + +# TLS +tls_certificate = "../../.circleci/server.cert" +tls_private_key = "../../.circleci/server.key" + +# Credentials to access the virtual administrative database (pgbouncer or pgcat) +# Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. +admin_username = "admin_user" +admin_password = "admin_pass" + +# pool +# configs are structured as pool. +# the pool_name is what clients use as database name when connecting +# For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded_db" +[pools.sharded_db] +# Pool mode (see PgBouncer docs for more). +# session: one server connection per connected client +# transaction: one server connection per client transaction +pool_mode = "transaction" + +# If the client doesn't specify, route traffic to +# this role by default. +# +# any: round-robin between primary and replicas, +# replica: round-robin between replicas only without touching the primary, +# primary: all queries go to the primary unless otherwise specified. +default_role = "any" + +# Query parser. If enabled, we'll attempt to parse +# every incoming query to determine if it's a read or a write. +# If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, +# we'll direct it to the primary. +query_parser_enabled = true + +# If the query parser is enabled and this setting is enabled, we'll attempt to +# infer the role from the query itself. +query_parser_read_write_splitting = true + +# If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for +# load balancing of read queries. Otherwise, the primary will only be used for write +# queries. The primary can always be explicitely selected with our custom protocol. +primary_reads_enabled = true + +# So what if you wanted to implement a different hashing function, +# or you've already built one and you want this pooler to use it? +# +# Current options: +# +# pg_bigint_hash: PARTITION BY HASH (Postgres hashing function) +# sha1: A hashing function based on SHA1 +# +sharding_function = "pg_bigint_hash" + +# Prepared statements cache size. +prepared_statements_cache_size = 500 + +# Credentials for users that may connect to this cluster +[pools.sharded_db.users.0] +username = "sharding_user" +password = "sharding_user" +# Maximum number of server connections that can be established for this user +# The maximum number of connection from a single Pgcat process to any database in the cluster +# is the sum of pool_size across all users. +pool_size = 5 +statement_timeout = 0 + + +[pools.sharded_db.users.1] +username = "other_user" +password = "other_user" +pool_size = 21 +statement_timeout = 30000 + +# Shard 0 +[pools.sharded_db.shards.0] +# [ host, port, role ] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ] +] +# Database name (e.g. "postgres") +database = "shard0" + +[pools.sharded_db.shards.1] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ], +] +database = "shard1" + +[pools.sharded_db.shards.2] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ], +] +database = "shard2" + + +[pools.simple_db] +pool_mode = "session" +default_role = "primary" +query_parser_enabled = true +query_parser_read_write_splitting = true +primary_reads_enabled = true +sharding_function = "pg_bigint_hash" + +[pools.simple_db.users.0] +username = "simple_user" +password = "simple_user" +pool_size = 5 +statement_timeout = 30000 + +[pools.simple_db.shards.0] +servers = [ + [ "127.0.0.1", 5432, "primary" ], + [ "localhost", 5432, "replica" ] +] +database = "some_db" diff --git a/tests/go/prepared_test.go b/tests/go/prepared_test.go new file mode 100644 index 000000000..0a42e721a --- /dev/null +++ b/tests/go/prepared_test.go @@ -0,0 +1,52 @@ +package pgcat + +import ( + "context" + "database/sql" + "fmt" + _ "github.com/lib/pq" + "testing" +) + +func Test(t *testing.T) { + t.Cleanup(setup(t)) + t.Run("Named parameterized prepared statement works", namedParameterizedPreparedStatement) + t.Run("Unnamed parameterized prepared statement works", unnamedParameterizedPreparedStatement) +} + +func namedParameterizedPreparedStatement(t *testing.T) { + db, err := sql.Open("postgres", fmt.Sprintf("host=localhost port=%d database=sharded_db user=sharding_user password=sharding_user sslmode=disable", port)) + if err != nil { + t.Fatalf("could not open connection: %+v", err) + } + + stmt, err := db.Prepare("SELECT $1") + + if err != nil { + t.Fatalf("could not prepare: %+v", err) + } + + for i := 0; i < 100; i++ { + rows, err := stmt.Query(1) + if err != nil { + t.Fatalf("could not query: %+v", err) + } + _ = rows.Close() + } +} + +func unnamedParameterizedPreparedStatement(t *testing.T) { + db, err := sql.Open("postgres", fmt.Sprintf("host=localhost port=%d database=sharded_db user=sharding_user password=sharding_user sslmode=disable", port)) + if err != nil { + t.Fatalf("could not open connection: %+v", err) + } + + for i := 0; i < 100; i++ { + // Under the hood QueryContext generates an unnamed parameterized prepared statement + rows, err := db.QueryContext(context.Background(), "SELECT $1", 1) + if err != nil { + t.Fatalf("could not query: %+v", err) + } + _ = rows.Close() + } +} diff --git a/tests/go/setup.go b/tests/go/setup.go new file mode 100644 index 000000000..32ffc4ba2 --- /dev/null +++ b/tests/go/setup.go @@ -0,0 +1,81 @@ +package pgcat + +import ( + "context" + "database/sql" + _ "embed" + "fmt" + "math/rand" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +//go:embed pgcat.toml +var pgcatCfg string + +var port = rand.Intn(32760-20000) + 20000 + +func setup(t *testing.T) func() { + cfg, err := os.CreateTemp("/tmp", "pgcat_cfg_*.toml") + if err != nil { + t.Fatalf("could not create temp file: %+v", err) + } + + pgcatCfg = strings.Replace(pgcatCfg, "\"${PORT}\"", fmt.Sprintf("%d", port), 1) + + _, err = cfg.Write([]byte(pgcatCfg)) + if err != nil { + t.Fatalf("could not write temp file: %+v", err) + } + + commandPath := "../../target/debug/pgcat" + if os.Getenv("CARGO_TARGET_DIR") != "" { + commandPath = os.Getenv("CARGO_TARGET_DIR") + "/debug/pgcat" + } + + cmd := exec.Command(commandPath, cfg.Name()) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + go func() { + err = cmd.Run() + if err != nil { + t.Errorf("could not run pgcat: %+v", err) + } + }() + + deadline, cancelFunc := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second)) + defer cancelFunc() + for { + select { + case <-deadline.Done(): + break + case <-time.After(50 * time.Millisecond): + db, err := sql.Open("postgres", fmt.Sprintf("host=localhost port=%d database=pgcat user=admin_user password=admin_pass sslmode=disable", port)) + if err != nil { + continue + } + rows, err := db.QueryContext(deadline, "SHOW STATS") + if err != nil { + continue + } + _ = rows.Close() + _ = db.Close() + break + } + break + } + + return func() { + err := cmd.Process.Signal(os.Interrupt) + if err != nil { + t.Fatalf("could not interrupt pgcat: %+v", err) + } + err = os.Remove(cfg.Name()) + if err != nil { + t.Fatalf("could not remove temp file: %+v", err) + } + } +} From 9eb5bfae6516fc5af7c5efe0dd8b67c44372f6f4 Mon Sep 17 00:00:00 2001 From: Calvin Hughes Date: Mon, 13 Nov 2023 19:24:39 +0000 Subject: [PATCH 483/551] Expose clients maxwait time in SHOW CLIENTS response via admin (#639) * Expose clients maxwait time in SHOW CLIENTS response via PgCat admin Displays the maxwait via maxwait_seconds and maxwait_us columns for each client that can be used to track down the wait time per client in a case where the overall pool stats shows waiting time. The maxwait_us, similar to the pool stats setup, is configured to display as a remainder alongside the maxwait_seconds. * Use maxwait instead of maxwait_seconds to match pools column name --------- --- src/admin.rs | 5 +++++ tests/ruby/stats_spec.rb | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/admin.rs b/src/admin.rs index 80baa3fba..22bbb0af1 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -690,6 +690,8 @@ where ("query_count", DataType::Numeric), ("error_count", DataType::Numeric), ("age_seconds", DataType::Numeric), + ("maxwait", DataType::Numeric), + ("maxwait_us", DataType::Numeric), ]; let new_map = get_client_stats(); @@ -697,6 +699,7 @@ where res.put(row_description(&columns)); for (_, client) in new_map { + let max_wait = client.max_wait_time.load(Ordering::Relaxed); let row = vec![ format!("{:#010X}", client.client_id()), client.pool_name(), @@ -710,6 +713,8 @@ where .duration_since(client.connect_time()) .as_secs() .to_string(), + (max_wait / 1_000_000).to_string(), + (max_wait % 1_000_000).to_string(), ]; res.put(data_row(&row)); diff --git a/tests/ruby/stats_spec.rb b/tests/ruby/stats_spec.rb index 132b80a9e..ddf63cd3e 100644 --- a/tests/ruby/stats_spec.rb +++ b/tests/ruby/stats_spec.rb @@ -329,6 +329,40 @@ admin_conn.close connections.map(&:close) end + + context "when client has waited for a server" do + let(:processes) { Helpers::Pgcat.single_instance_setup("sharded_db", 2) } + + it "shows correct maxwait" do + threads = [] + connections = Array.new(3) { |i| PG::connect("#{pgcat_conn_str}?application_name=app#{i}") } + connections.each do |c| + threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") rescue nil } + end + + sleep(2.5) # Allow time for stats to update + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + results = admin_conn.async_exec("SHOW CLIENTS") + + normal_client_results = results.reject { |r| r["database"] == "pgcat" } + + non_waiting_clients = normal_client_results.select { |c| c["maxwait"] == "0" } + waiting_clients = normal_client_results.select { |c| c["maxwait"].to_i > 0 } + + expect(non_waiting_clients.count).to eq(2) + non_waiting_clients.each do |client| + expect(client["maxwait_us"].to_i).to be_between(0, 50_000) + end + + expect(waiting_clients.count).to eq(1) + waiting_clients.each do |client| + expect(client["maxwait_us"].to_i).to be_within(200_000).of(500_000) + end + + admin_conn.close + connections.map(&:close) + end + end end From 23677b82d225ad44f0baddf9ecca0f7a20473010 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 28 Nov 2023 21:13:30 -0800 Subject: [PATCH 484/551] Dont cache prepared statement with errors (#647) * Fix prepared statement not found when prepared stmt has error * cleanup debug * remove more debug msgs * sure debugged this.. * version bump * add rust tests --- .circleci/run_tests.sh | 7 +++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- src/client.rs | 35 +++++++++++++++++++++++++++----- src/errors.rs | 1 + src/server.rs | 45 +++++++++++++++++++++++++++++++++++++----- tests/rust/src/main.rs | 9 ++++++++- 7 files changed, 88 insertions(+), 13 deletions(-) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 3a31240af..b6bc42281 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -119,6 +119,13 @@ popd start_pgcat "info" +# +# Rust tests +# +cd tests/rust +cargo run +cd ../../ + # Admin tests export PGPASSWORD=admin_pass psql -U admin_user -e -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS' > /dev/null diff --git a/Cargo.lock b/Cargo.lock index e50439758..28f20a9da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,7 +1020,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pgcat" -version = "1.1.2-dev2" +version = "1.1.2-dev4" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 6485622a0..f451ffc5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.1.2-dev2" +version = "1.1.2-dev4" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/src/client.rs b/src/client.rs index dd89697c1..bbeb5265a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1149,7 +1149,7 @@ where // This reads the first byte without advancing the internal pointer and mutating the bytes let code = *message.first().unwrap() as char; - trace!("Message: {}", code); + trace!("Client message: {}", code); match code { // Query @@ -1188,6 +1188,7 @@ where }; } } + debug!("Sending query to server"); self.send_and_receive_loop( @@ -1320,6 +1321,7 @@ where { match protocol_data { ExtendedProtocolData::Parse { data, metadata } => { + debug!("Have parse in extended buffer"); let (parse, hash) = match metadata { Some(metadata) => metadata, None => { @@ -1656,11 +1658,25 @@ where ) -> Result<(), Error> { match self.prepared_statements.get(&client_name) { Some((parse, hash)) => { - debug!("Prepared statement `{}` found in cache", parse.name); + debug!("Prepared statement `{}` found in cache", client_name); // In this case we want to send the parse message to the server // since pgcat is initiating the prepared statement on this specific server - self.register_parse_to_server_cache(true, hash, parse, pool, server, address) - .await?; + match self + .register_parse_to_server_cache(true, hash, parse, pool, server, address) + .await + { + Ok(_) => (), + Err(err) => match err { + Error::PreparedStatementError => { + debug!("Removed {} from client cache", client_name); + self.prepared_statements.remove(&client_name); + } + + _ => { + return Err(err); + } + }, + } } None => { @@ -1689,11 +1705,20 @@ where // We want to promote this in the pool's LRU pool.promote_prepared_statement_hash(hash); + debug!("Checking for prepared statement {}", parse.name); + if let Err(err) = server .register_prepared_statement(parse, should_send_parse_to_server) .await { - pool.ban(address, BanReason::MessageSendFailed, Some(&self.stats)); + match err { + // Don't ban for this. + Error::PreparedStatementError => (), + _ => { + pool.ban(address, BanReason::MessageSendFailed, Some(&self.stats)); + } + }; + return Err(err); } diff --git a/src/errors.rs b/src/errors.rs index a6aebc509..13047b4bd 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -29,6 +29,7 @@ pub enum Error { QueryRouterParserError(String), QueryRouterError(String), InvalidShardId(usize), + PreparedStatementError, } #[derive(Clone, PartialEq, Debug)] diff --git a/src/server.rs b/src/server.rs index dff6a7694..9089b56ff 100644 --- a/src/server.rs +++ b/src/server.rs @@ -7,7 +7,7 @@ use lru::LruCache; use once_cell::sync::Lazy; use parking_lot::{Mutex, RwLock}; use postgres_protocol::message; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::mem; use std::net::IpAddr; use std::num::NonZeroUsize; @@ -325,6 +325,9 @@ pub struct Server { /// Prepared statements prepared_statement_cache: Option>, + + /// Prepared statement being currently registered on the server. + registering_prepared_statement: VecDeque, } impl Server { @@ -827,6 +830,7 @@ impl Server { NonZeroUsize::new(prepared_statement_cache_size).unwrap(), )), }, + registering_prepared_statement: VecDeque::new(), }; return Ok(server); @@ -956,7 +960,6 @@ impl Server { // There is no more data available from the server. self.data_available = false; - break; } @@ -966,6 +969,23 @@ impl Server { self.in_copy_mode = false; } + // Remove the prepared statement from the cache, it has a syntax error or something else bad happened. + if let Some(prepared_stmt_name) = + self.registering_prepared_statement.pop_front() + { + if let Some(ref mut cache) = self.prepared_statement_cache { + if let Some(_removed) = cache.pop(&prepared_stmt_name) { + debug!( + "Removed {} from prepared statement cache", + prepared_stmt_name + ); + } else { + // Shouldn't happen. + debug!("Prepared statement {} was not cached", prepared_stmt_name); + } + } + } + if self.prepared_statement_cache.is_some() { let error_message = PgErrorMsg::parse(&message)?; if error_message.message == "cached plan must not change result type" { @@ -1068,6 +1088,11 @@ impl Server { // Buffer until ReadyForQuery shows up, so don't exit the loop yet. 'c' => (), + // Parse complete successfully + '1' => { + self.registering_prepared_statement.pop_front(); + } + // Anything else, e.g. errors, notices, etc. // Keep buffering until ReadyForQuery shows up. _ => (), @@ -1107,7 +1132,7 @@ impl Server { has_it } - pub fn add_prepared_statement_to_cache(&mut self, name: &str) -> Option { + fn add_prepared_statement_to_cache(&mut self, name: &str) -> Option { let cache = match &mut self.prepared_statement_cache { Some(cache) => cache, None => return None, @@ -1129,7 +1154,7 @@ impl Server { None } - pub fn remove_prepared_statement_from_cache(&mut self, name: &str) { + fn remove_prepared_statement_from_cache(&mut self, name: &str) { let cache = match &mut self.prepared_statement_cache { Some(cache) => cache, None => return, @@ -1145,6 +1170,9 @@ impl Server { should_send_parse_to_server: bool, ) -> Result<(), Error> { if !self.has_prepared_statement(&parse.name) { + self.registering_prepared_statement + .push_back(parse.name.clone()); + let mut bytes = BytesMut::new(); if should_send_parse_to_server { @@ -1176,7 +1204,13 @@ impl Server { } }; - Ok(()) + // If it's not there, something went bad, I'm guessing bad syntax or permissions error + // on the server. + if !self.has_prepared_statement(&parse.name) { + Err(Error::PreparedStatementError) + } else { + Ok(()) + } } /// If the server is still inside a transaction. @@ -1186,6 +1220,7 @@ impl Server { self.in_transaction } + /// Currently copying data from client to server or vice-versa. pub fn in_copy_mode(&self) -> bool { self.in_copy_mode } diff --git a/tests/rust/src/main.rs b/tests/rust/src/main.rs index 79667bcc8..c61d48ccd 100644 --- a/tests/rust/src/main.rs +++ b/tests/rust/src/main.rs @@ -16,7 +16,14 @@ async fn test_prepared_statements() { let pool = pool.clone(); let handle = tokio::task::spawn(async move { for _ in 0..1000 { - sqlx::query("SELECT 1").fetch_all(&pool).await.unwrap(); + match sqlx::query("SELECT one").fetch_all(&pool).await { + Ok(_) => (), + Err(err) => { + if err.to_string().contains("prepared statement") { + panic!("prepared statement error: {}", err); + } + } + } } }); From af63941d8fe7dfd15976a0fec3196cfc16e35d0e Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 3 Dec 2023 20:27:44 -0800 Subject: [PATCH 485/551] Fix deb build overwriting config (#651) --- utilities/deb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/deb.sh b/utilities/deb.sh index b01a9929e..4aa8aa781 100644 --- a/utilities/deb.sh +++ b/utilities/deb.sh @@ -22,7 +22,7 @@ mkdir -p "$deb_dir/etc/systemd/system" cp target/release/pgcat "$deb_dir/usr/bin/pgcat" chmod +x "$deb_dir/usr/bin/pgcat" -cp pgcat.toml "$deb_dir/etc/pgcat.toml" +cp pgcat.toml "$deb_dir/etc/pgcat.example.toml" cp pgcat.service "$deb_dir/etc/systemd/system/pgcat.service" (cat control | envsubst) > "$deb_dir/DEBIAN/control" From 3402804c840d8540c57717f02fea9b1fafaf92e5 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 3 Dec 2023 20:58:39 -0800 Subject: [PATCH 486/551] Broken blog link (#652) Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ae310cded..ad6e86035 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ PgCat is stable and used in production to serve hundreds of thousands of queries - + PostgresML From 87065313a0e1e38f5d6d5b0a2e264fba559caa5f Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Sun, 3 Dec 2023 21:01:23 -0800 Subject: [PATCH 487/551] Broken blog link 2 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ad6e86035..2925fb04e 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ PgCat is stable and used in production to serve hundreds of thousands of queries - + From e8052461c399781ecec389a377a1df3900c7abac Mon Sep 17 00:00:00 2001 From: Alec Date: Mon, 4 Dec 2023 19:09:41 -0500 Subject: [PATCH 488/551] Require a reason when marking a server bad (#654) When calling mark_bad require a reason so it can be logged rather than the generic message --- src/client.rs | 16 +++++++++------- src/mirrors.rs | 10 ++++++---- src/pool.rs | 2 +- src/server.rs | 4 ++-- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/client.rs b/src/client.rs index bbeb5265a..23392b73a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1437,7 +1437,7 @@ where .await { // We might be in some kind of error/in between protocol state - server.mark_bad(); + server.mark_bad(err.to_string().as_str()); return Err(err); } @@ -1504,7 +1504,7 @@ where match write_all_flush(&mut self.write, &response).await { Ok(_) => (), Err(err) => { - server.mark_bad(); + server.mark_bad(err.to_string().as_str()); return Err(err); } }; @@ -1926,7 +1926,7 @@ where Ok(_) => (), Err(err) => { // We might be in some kind of error/in between protocol state, better to just kill this server - server.mark_bad(); + server.mark_bad(err.to_string().as_str()); return Err(err); } }; @@ -1993,11 +1993,13 @@ where } }, Err(_) => { - error!( - "Statement timeout while talking to {:?} with user {}", - address, pool.settings.user.username + server.mark_bad( + format!( + "Statement timeout while talking to {:?} with user {}", + address, pool.settings.user.username + ) + .as_str(), ); - server.mark_bad(); pool.ban(address, BanReason::StatementTimeout, Some(client_stats)); error_response_terminal(&mut self.write, "pool statement timeout").await?; Err(Error::StatementTimeout) diff --git a/src/mirrors.rs b/src/mirrors.rs index 5c39504e7..73ab73207 100644 --- a/src/mirrors.rs +++ b/src/mirrors.rs @@ -85,8 +85,9 @@ impl MirroredClient { match recv_result { Ok(message) => trace!("Received from mirror: {} {:?}", String::from_utf8_lossy(&message[..]), address.clone()), Err(err) => { - server.mark_bad(); - error!("Failed to receive from mirror {:?} {:?}", err, address.clone()); + server.mark_bad( + format!("Failed to send to mirror, Discarding message {:?}, {:?}", err, address.clone()).as_str() + ); } } } @@ -98,8 +99,9 @@ impl MirroredClient { match server.send(&BytesMut::from(&bytes[..])).await { Ok(_) => trace!("Sent to mirror: {} {:?}", String::from_utf8_lossy(&bytes[..]), address.clone()), Err(err) => { - server.mark_bad(); - error!("Failed to send to mirror, Discarding message {:?}, {:?}", err, address.clone()) + server.mark_bad( + format!("Failed to receive from mirror {:?} {:?}", err, address.clone()).as_str() + ); } } } diff --git a/src/pool.rs b/src/pool.rs index 751f18759..00f4dc2a6 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -871,7 +871,7 @@ impl ConnectionPool { } // Don't leave a bad connection in the pool. - server.mark_bad(); + server.mark_bad("failed health check"); self.ban(address, BanReason::FailedHealthCheck, Some(client_info)); false diff --git a/src/server.rs b/src/server.rs index 9089b56ff..882450eaf 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1279,8 +1279,8 @@ impl Server { } /// Indicate that this server connection cannot be re-used and must be discarded. - pub fn mark_bad(&mut self) { - error!("Server {:?} marked bad", self.address); + pub fn mark_bad(&mut self, reason: &str) { + error!("Server {:?} marked bad, reason: {}", self.address, reason); self.bad = true; } From a4d6fee757ffb7b280849e7fae2a532145110433 Mon Sep 17 00:00:00 2001 From: Daniel Babiak Date: Mon, 4 Dec 2023 21:19:51 -0500 Subject: [PATCH 489/551] only report wait times from clients currently waiting to match behavior of pgbouncer (#655) * Change maxwait to only report wait times from clients currently waiting to match behavior of pgbouncer * Fix tests --- src/stats/client.rs | 15 ++++++++++----- src/stats/pool.rs | 14 +++++++++++--- tests/ruby/stats_spec.rb | 16 ++++++++++++---- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/stats/client.rs b/src/stats/client.rs index 6a30ec15a..0079fbada 100644 --- a/src/stats/client.rs +++ b/src/stats/client.rs @@ -38,8 +38,10 @@ pub struct ClientStats { /// Total time spent waiting for a connection from pool, measures in microseconds pub total_wait_time: Arc, - /// Maximum time spent waiting for a connection from pool, measures in microseconds - pub max_wait_time: Arc, + /// When this client started waiting. + /// Stored as microseconds since connect_time so it can fit in an AtomicU64 instead + /// of us using an "AtomicInstant" + pub wait_start: Arc, /// Current state of the client pub state: Arc, @@ -63,7 +65,7 @@ impl Default for ClientStats { username: String::new(), pool_name: String::new(), total_wait_time: Arc::new(AtomicU64::new(0)), - max_wait_time: Arc::new(AtomicU64::new(0)), + wait_start: Arc::new(AtomicU64::new(0)), state: Arc::new(AtomicClientState::new(ClientState::Idle)), transaction_count: Arc::new(AtomicU64::new(0)), query_count: Arc::new(AtomicU64::new(0)), @@ -111,6 +113,11 @@ impl ClientStats { /// Reports a client is waiting for a connection pub fn waiting(&self) { + // safe to truncate, we only lose info if duration is greater than ~585,000 years + self.wait_start.store( + Instant::now().duration_since(self.connect_time).as_micros() as u64, + Ordering::Relaxed, + ); self.state.store(ClientState::Waiting, Ordering::Relaxed); } @@ -134,8 +141,6 @@ impl ClientStats { pub fn checkout_time(&self, microseconds: u64) { self.total_wait_time .fetch_add(microseconds, Ordering::Relaxed); - self.max_wait_time - .fetch_max(microseconds, Ordering::Relaxed); } /// Report a query executed by a client against a server diff --git a/src/stats/pool.rs b/src/stats/pool.rs index 46c746329..a3724bdd5 100644 --- a/src/stats/pool.rs +++ b/src/stats/pool.rs @@ -4,6 +4,7 @@ use super::{ClientState, ServerState}; use crate::{config::PoolMode, messages::DataType, pool::PoolIdentifier}; use std::collections::HashMap; use std::sync::atomic::*; +use tokio::time::Instant; use crate::pool::get_all_pools; @@ -53,6 +54,7 @@ impl PoolStats { ); } + let now = Instant::now(); for client in client_map.values() { match map.get_mut(&PoolIdentifier { db: client.pool_name(), @@ -62,10 +64,16 @@ impl PoolStats { match client.state.load(Ordering::Relaxed) { ClientState::Active => pool_stats.cl_active += 1, ClientState::Idle => pool_stats.cl_idle += 1, - ClientState::Waiting => pool_stats.cl_waiting += 1, + ClientState::Waiting => { + pool_stats.cl_waiting += 1; + // wait_start is measured as microseconds since connect_time + // so compute wait_time as (now() - connect_time) - (wait_start - connect_time) + let duration_since_connect = now.duration_since(client.connect_time()); + let wait_time = (duration_since_connect.as_micros() as u64) + - client.wait_start.load(Ordering::Relaxed); + pool_stats.maxwait = std::cmp::max(pool_stats.maxwait, wait_time); + } } - let max_wait = client.max_wait_time.load(Ordering::Relaxed); - pool_stats.maxwait = std::cmp::max(pool_stats.maxwait, max_wait); } None => debug!("Client from an obselete pool"), } diff --git a/tests/ruby/stats_spec.rb b/tests/ruby/stats_spec.rb index ddf63cd3e..2d635f0a3 100644 --- a/tests/ruby/stats_spec.rb +++ b/tests/ruby/stats_spec.rb @@ -233,7 +233,7 @@ sleep(1.1) # Allow time for stats to update admin_conn = PG::connect(processes.pgcat.admin_connection_string) results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s| + %w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login].each do |s| raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" end @@ -260,12 +260,20 @@ threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") rescue nil } end - sleep(2.5) # Allow time for stats to update admin_conn = PG::connect(processes.pgcat.admin_connection_string) - results = admin_conn.async_exec("SHOW POOLS")[0] + # two connections waiting => they report wait time + sleep(1.1) # Allow time for stats to update + results = admin_conn.async_exec("SHOW POOLS")[0] expect(results["maxwait"]).to eq("1") - expect(results["maxwait_us"].to_i).to be_within(200_000).of(500_000) + expect(results["maxwait_us"].to_i).to be_within(200_000).of(100_000) + + sleep(2.5) # Allow time for stats to update + results = admin_conn.async_exec("SHOW POOLS")[0] + + # no connections waiting => no reported wait time + expect(results["maxwait"]).to eq("0") + expect(results["maxwait_us"]).to eq("0") connections.map(&:close) sleep(4.5) # Allow time for stats to update From 6f8ede189749caa4956404b1163010219576d8e3 Mon Sep 17 00:00:00 2001 From: Lev Date: Mon, 4 Dec 2023 18:30:03 -0800 Subject: [PATCH 490/551] Not sure how this sneaked past CI --- src/admin.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin.rs b/src/admin.rs index 22bbb0af1..f7be8665f 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -699,7 +699,7 @@ where res.put(row_description(&columns)); for (_, client) in new_map { - let max_wait = client.max_wait_time.load(Ordering::Relaxed); + let max_wait = client.wait_start.load(Ordering::Relaxed); let row = vec![ format!("{:#010X}", client.client_id()), client.pool_name(), From fe271cdc04e5c36746635d11c924cc992d5a82d1 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 4 Dec 2023 18:33:08 -0800 Subject: [PATCH 491/551] Reset wait times when checked out successfully (#656) --- src/stats/client.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/stats/client.rs b/src/stats/client.rs index 0079fbada..c3a636d0a 100644 --- a/src/stats/client.rs +++ b/src/stats/client.rs @@ -109,6 +109,7 @@ impl ClientStats { /// Reports a client is done querying the server and is no longer assigned a server connection pub fn idle(&self) { self.state.store(ClientState::Idle, Ordering::Relaxed); + self.wait_start.store(0, Ordering::Relaxed); } /// Reports a client is waiting for a connection @@ -124,6 +125,7 @@ impl ClientStats { /// Reports a client is done waiting for a connection and is about to query the server. pub fn active(&self) { self.state.store(ClientState::Active, Ordering::Relaxed); + self.wait_start.store(0, Ordering::Relaxed); } /// Reports a client has failed to obtain a connection from a connection pool From 411747c22c4b15b6f4ebd5167de4683dd6c1552b Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 5 Dec 2023 01:47:38 -0800 Subject: [PATCH 492/551] Revert max_wait changes (#658) * Revert "Reset wait times when checked out successfully (#656)" This reverts commit fe271cdc04e5c36746635d11c924cc992d5a82d1. * Revert "Not sure how this sneaked past CI" This reverts commit 6f8ede189749caa4956404b1163010219576d8e3. * Revert "only report wait times from clients currently waiting to match behavior of pgbouncer (#655)" This reverts commit a4d6fee757ffb7b280849e7fae2a532145110433. --- src/admin.rs | 2 +- src/stats/client.rs | 17 +++++------------ src/stats/pool.rs | 14 +++----------- tests/ruby/stats_spec.rb | 16 ++++------------ 4 files changed, 13 insertions(+), 36 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index f7be8665f..22bbb0af1 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -699,7 +699,7 @@ where res.put(row_description(&columns)); for (_, client) in new_map { - let max_wait = client.wait_start.load(Ordering::Relaxed); + let max_wait = client.max_wait_time.load(Ordering::Relaxed); let row = vec![ format!("{:#010X}", client.client_id()), client.pool_name(), diff --git a/src/stats/client.rs b/src/stats/client.rs index c3a636d0a..6a30ec15a 100644 --- a/src/stats/client.rs +++ b/src/stats/client.rs @@ -38,10 +38,8 @@ pub struct ClientStats { /// Total time spent waiting for a connection from pool, measures in microseconds pub total_wait_time: Arc, - /// When this client started waiting. - /// Stored as microseconds since connect_time so it can fit in an AtomicU64 instead - /// of us using an "AtomicInstant" - pub wait_start: Arc, + /// Maximum time spent waiting for a connection from pool, measures in microseconds + pub max_wait_time: Arc, /// Current state of the client pub state: Arc, @@ -65,7 +63,7 @@ impl Default for ClientStats { username: String::new(), pool_name: String::new(), total_wait_time: Arc::new(AtomicU64::new(0)), - wait_start: Arc::new(AtomicU64::new(0)), + max_wait_time: Arc::new(AtomicU64::new(0)), state: Arc::new(AtomicClientState::new(ClientState::Idle)), transaction_count: Arc::new(AtomicU64::new(0)), query_count: Arc::new(AtomicU64::new(0)), @@ -109,23 +107,16 @@ impl ClientStats { /// Reports a client is done querying the server and is no longer assigned a server connection pub fn idle(&self) { self.state.store(ClientState::Idle, Ordering::Relaxed); - self.wait_start.store(0, Ordering::Relaxed); } /// Reports a client is waiting for a connection pub fn waiting(&self) { - // safe to truncate, we only lose info if duration is greater than ~585,000 years - self.wait_start.store( - Instant::now().duration_since(self.connect_time).as_micros() as u64, - Ordering::Relaxed, - ); self.state.store(ClientState::Waiting, Ordering::Relaxed); } /// Reports a client is done waiting for a connection and is about to query the server. pub fn active(&self) { self.state.store(ClientState::Active, Ordering::Relaxed); - self.wait_start.store(0, Ordering::Relaxed); } /// Reports a client has failed to obtain a connection from a connection pool @@ -143,6 +134,8 @@ impl ClientStats { pub fn checkout_time(&self, microseconds: u64) { self.total_wait_time .fetch_add(microseconds, Ordering::Relaxed); + self.max_wait_time + .fetch_max(microseconds, Ordering::Relaxed); } /// Report a query executed by a client against a server diff --git a/src/stats/pool.rs b/src/stats/pool.rs index a3724bdd5..46c746329 100644 --- a/src/stats/pool.rs +++ b/src/stats/pool.rs @@ -4,7 +4,6 @@ use super::{ClientState, ServerState}; use crate::{config::PoolMode, messages::DataType, pool::PoolIdentifier}; use std::collections::HashMap; use std::sync::atomic::*; -use tokio::time::Instant; use crate::pool::get_all_pools; @@ -54,7 +53,6 @@ impl PoolStats { ); } - let now = Instant::now(); for client in client_map.values() { match map.get_mut(&PoolIdentifier { db: client.pool_name(), @@ -64,16 +62,10 @@ impl PoolStats { match client.state.load(Ordering::Relaxed) { ClientState::Active => pool_stats.cl_active += 1, ClientState::Idle => pool_stats.cl_idle += 1, - ClientState::Waiting => { - pool_stats.cl_waiting += 1; - // wait_start is measured as microseconds since connect_time - // so compute wait_time as (now() - connect_time) - (wait_start - connect_time) - let duration_since_connect = now.duration_since(client.connect_time()); - let wait_time = (duration_since_connect.as_micros() as u64) - - client.wait_start.load(Ordering::Relaxed); - pool_stats.maxwait = std::cmp::max(pool_stats.maxwait, wait_time); - } + ClientState::Waiting => pool_stats.cl_waiting += 1, } + let max_wait = client.max_wait_time.load(Ordering::Relaxed); + pool_stats.maxwait = std::cmp::max(pool_stats.maxwait, max_wait); } None => debug!("Client from an obselete pool"), } diff --git a/tests/ruby/stats_spec.rb b/tests/ruby/stats_spec.rb index 2d635f0a3..ddf63cd3e 100644 --- a/tests/ruby/stats_spec.rb +++ b/tests/ruby/stats_spec.rb @@ -233,7 +233,7 @@ sleep(1.1) # Allow time for stats to update admin_conn = PG::connect(processes.pgcat.admin_connection_string) results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login].each do |s| + %w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s| raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" end @@ -260,20 +260,12 @@ threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") rescue nil } end - admin_conn = PG::connect(processes.pgcat.admin_connection_string) - - # two connections waiting => they report wait time - sleep(1.1) # Allow time for stats to update - results = admin_conn.async_exec("SHOW POOLS")[0] - expect(results["maxwait"]).to eq("1") - expect(results["maxwait_us"].to_i).to be_within(200_000).of(100_000) - sleep(2.5) # Allow time for stats to update + admin_conn = PG::connect(processes.pgcat.admin_connection_string) results = admin_conn.async_exec("SHOW POOLS")[0] - # no connections waiting => no reported wait time - expect(results["maxwait"]).to eq("0") - expect(results["maxwait_us"]).to eq("0") + expect(results["maxwait"]).to eq("1") + expect(results["maxwait_us"].to_i).to be_within(200_000).of(500_000) connections.map(&:close) sleep(4.5) # Allow time for stats to update From 1bf21103de0cbf11340dc52a6088a848152f1187 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 18 Jan 2024 11:57:28 -0600 Subject: [PATCH 493/551] Report waiting time only for currently waiting clients (#678) The pool maxwait metric currently operates differently from Pgbouncer. The way it operates today is that we keep track of max_wait on each connected client, when SHOW POOLS query is made, we go over the connected clients and we get the max of max_wait times among clients. This means the pool maxwait will never reset, it will always be monotonically increasing until the client with the highest maxwait disconnects. This PR changes this behavior, by keeping track of the wait_start time on each client, when a client goes into WAITING state, we record the time offset from connect_time. When we either successfully or unsuccessfully checkout a connection from the pool, we reset the wait_start time. When SHOW POOLS query is made, we go over all connected clients and we only consider clients whose wait_start is non-zero, for clients that have non-zero wait times, we compare them and report the maximum waiting time as maxwait for the pool. --- src/pool.rs | 12 ++++-------- src/stats/client.rs | 38 ++++++++++++++++++++++++++++++++++---- src/stats/pool.rs | 7 +++++-- tests/ruby/stats_spec.rb | 21 ++++++++++++--------- 4 files changed, 55 insertions(+), 23 deletions(-) diff --git a/src/pool.rs b/src/pool.rs index 00f4dc2a6..c03aaf4c9 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -769,7 +769,6 @@ impl ConnectionPool { ); self.ban(address, BanReason::FailedCheckout, Some(client_stats)); address.stats.error(); - client_stats.idle(); client_stats.checkout_error(); continue; } @@ -788,7 +787,7 @@ impl ConnectionPool { // Health checks are pretty expensive. if !require_healthcheck { let checkout_time = now.elapsed().as_micros() as u64; - client_stats.checkout_time(checkout_time); + client_stats.checkout_success(); server .stats() .checkout_time(checkout_time, client_stats.application_name()); @@ -802,7 +801,7 @@ impl ConnectionPool { .await { let checkout_time = now.elapsed().as_micros() as u64; - client_stats.checkout_time(checkout_time); + client_stats.checkout_success(); server .stats() .checkout_time(checkout_time, client_stats.application_name()); @@ -814,10 +813,7 @@ impl ConnectionPool { } } - client_stats.idle(); - - let checkout_time = now.elapsed().as_micros() as u64; - client_stats.checkout_time(checkout_time); + client_stats.checkout_success(); Err(Error::AllServersDown) } @@ -843,7 +839,7 @@ impl ConnectionPool { Ok(res) => match res { Ok(_) => { let checkout_time: u64 = start.elapsed().as_micros() as u64; - client_info.checkout_time(checkout_time); + client_info.checkout_success(); server .stats() .checkout_time(checkout_time, client_info.application_name()); diff --git a/src/stats/client.rs b/src/stats/client.rs index 6a30ec15a..bd59a2f5c 100644 --- a/src/stats/client.rs +++ b/src/stats/client.rs @@ -41,6 +41,11 @@ pub struct ClientStats { /// Maximum time spent waiting for a connection from pool, measures in microseconds pub max_wait_time: Arc, + // Time when the client started waiting for a connection from pool, measures in microseconds + // We use connect_time as the reference point for this value + // U64 can represent ~5850 centuries in microseconds, so we should be fine + pub wait_start_us: Arc, + /// Current state of the client pub state: Arc, @@ -64,6 +69,7 @@ impl Default for ClientStats { pool_name: String::new(), total_wait_time: Arc::new(AtomicU64::new(0)), max_wait_time: Arc::new(AtomicU64::new(0)), + wait_start_us: Arc::new(AtomicU64::new(0)), state: Arc::new(AtomicClientState::new(ClientState::Idle)), transaction_count: Arc::new(AtomicU64::new(0)), query_count: Arc::new(AtomicU64::new(0)), @@ -111,6 +117,9 @@ impl ClientStats { /// Reports a client is waiting for a connection pub fn waiting(&self) { + let wait_start = self.connect_time.elapsed().as_micros() as u64; + + self.wait_start_us.store(wait_start, Ordering::Relaxed); self.state.store(ClientState::Waiting, Ordering::Relaxed); } @@ -122,6 +131,13 @@ impl ClientStats { /// Reports a client has failed to obtain a connection from a connection pool pub fn checkout_error(&self) { self.state.store(ClientState::Idle, Ordering::Relaxed); + self.update_wait_times(); + } + + /// Reports a client has succeeded in obtaining a connection from a connection pool + pub fn checkout_success(&self) { + self.state.store(ClientState::Active, Ordering::Relaxed); + self.update_wait_times(); } /// Reports a client has had the server assigned to it be banned @@ -130,12 +146,26 @@ impl ClientStats { self.error_count.fetch_add(1, Ordering::Relaxed); } - /// Reporters the time spent by a client waiting to get a healthy connection from the pool - pub fn checkout_time(&self, microseconds: u64) { + fn update_wait_times(&self) { + if self.wait_start_us.load(Ordering::Relaxed) == 0 { + return; + } + + let wait_time_us = self.get_current_wait_time_us(); self.total_wait_time - .fetch_add(microseconds, Ordering::Relaxed); + .fetch_add(wait_time_us, Ordering::Relaxed); self.max_wait_time - .fetch_max(microseconds, Ordering::Relaxed); + .fetch_max(wait_time_us, Ordering::Relaxed); + self.wait_start_us.store(0, Ordering::Relaxed); + } + + pub fn get_current_wait_time_us(&self) -> u64 { + let wait_start_us = self.wait_start_us.load(Ordering::Relaxed); + let microseconds_since_connection_epoch = self.connect_time.elapsed().as_micros() as u64; + if wait_start_us == 0 || microseconds_since_connection_epoch < wait_start_us { + return 0; + } + microseconds_since_connection_epoch - wait_start_us } /// Report a query executed by a client against a server diff --git a/src/stats/pool.rs b/src/stats/pool.rs index 46c746329..b5c6ff5bd 100644 --- a/src/stats/pool.rs +++ b/src/stats/pool.rs @@ -64,8 +64,11 @@ impl PoolStats { ClientState::Idle => pool_stats.cl_idle += 1, ClientState::Waiting => pool_stats.cl_waiting += 1, } - let max_wait = client.max_wait_time.load(Ordering::Relaxed); - pool_stats.maxwait = std::cmp::max(pool_stats.maxwait, max_wait); + let wait_start_us = client.wait_start_us.load(Ordering::Relaxed); + if wait_start_us > 0 { + let wait_time_us = client.get_current_wait_time_us(); + pool_stats.maxwait = std::cmp::max(pool_stats.maxwait, wait_time_us); + } } None => debug!("Client from an obselete pool"), } diff --git a/tests/ruby/stats_spec.rb b/tests/ruby/stats_spec.rb index ddf63cd3e..8a683a016 100644 --- a/tests/ruby/stats_spec.rb +++ b/tests/ruby/stats_spec.rb @@ -233,17 +233,19 @@ sleep(1.1) # Allow time for stats to update admin_conn = PG::connect(processes.pgcat.admin_connection_string) results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login maxwait].each do |s| + + %w[cl_idle cl_cancel_req sv_idle sv_used sv_tested sv_login].each do |s| raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" end + expect(results["maxwait"]).to eq("1") expect(results["cl_waiting"]).to eq("2") expect(results["cl_active"]).to eq("2") expect(results["sv_active"]).to eq("2") sleep(2.5) # Allow time for stats to update results = admin_conn.async_exec("SHOW POOLS")[0] - %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login].each do |s| + %w[cl_active cl_waiting cl_cancel_req sv_active sv_used sv_tested sv_login maxwait].each do |s| raise StandardError, "Field #{s} was expected to be 0 but found to be #{results[s]}" if results[s] != "0" end expect(results["cl_idle"]).to eq("4") @@ -255,22 +257,23 @@ it "show correct max_wait" do threads = [] + admin_conn = PG::connect(processes.pgcat.admin_connection_string) connections = Array.new(4) { PG::connect("#{pgcat_conn_str}?application_name=one_query") } connections.each do |c| threads << Thread.new { c.async_exec("SELECT pg_sleep(1.5)") rescue nil } end - - sleep(2.5) # Allow time for stats to update - admin_conn = PG::connect(processes.pgcat.admin_connection_string) + sleep(1.1) results = admin_conn.async_exec("SHOW POOLS")[0] - + # Value is only reported when there are clients waiting expect(results["maxwait"]).to eq("1") - expect(results["maxwait_us"].to_i).to be_within(200_000).of(500_000) - connections.map(&:close) + expect(results["maxwait_us"].to_i).to be_within(20_000).of(100_000) - sleep(4.5) # Allow time for stats to update + sleep(2.5) # Allow time for stats to update results = admin_conn.async_exec("SHOW POOLS")[0] + # no clients are waiting so value is 0 expect(results["maxwait"]).to eq("0") + expect(results["maxwait_us"]).to eq("0") + connections.map(&:close) threads.map(&:join) end From e6e5d35a9381c26d59433375a5087088c521bfed Mon Sep 17 00:00:00 2001 From: David ALEXANDRE <9482408+david972@users.noreply.github.com> Date: Thu, 22 Feb 2024 18:26:58 +0100 Subject: [PATCH 494/551] feat: add helm chart (#619) * add workflow * feat: add pgcat helm chart * fix: set the right include into configmap Signed-off-by: David ALEXANDRE * update values and config * prettifying config --------- Signed-off-by: David ALEXANDRE --- .github/dependabot.yml | 4 + .github/workflows/build-and-push.yaml | 4 +- .github/workflows/chart-lint-test.yaml | 50 +++ .github/workflows/chart-release.yaml | 40 ++ .github/workflows/generate-chart-readme.yaml | 48 +++ charts/pgcat/.helmignore | 23 ++ charts/pgcat/Chart.yaml | 8 + charts/pgcat/templates/NOTES.txt | 22 ++ charts/pgcat/templates/_config.tpl | 3 + charts/pgcat/templates/_helpers.tpl | 62 ++++ charts/pgcat/templates/deployment.yaml | 66 ++++ charts/pgcat/templates/ingress.yaml | 61 ++++ charts/pgcat/templates/secret.yaml | 86 +++++ charts/pgcat/templates/service.yaml | 15 + charts/pgcat/templates/serviceaccount.yaml | 12 + charts/pgcat/values.yaml | 361 +++++++++++++++++++ cr.yaml | 1 + ct.yaml | 5 + 18 files changed, 870 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/chart-lint-test.yaml create mode 100644 .github/workflows/chart-release.yaml create mode 100644 .github/workflows/generate-chart-readme.yaml create mode 100644 charts/pgcat/.helmignore create mode 100644 charts/pgcat/Chart.yaml create mode 100644 charts/pgcat/templates/NOTES.txt create mode 100644 charts/pgcat/templates/_config.tpl create mode 100644 charts/pgcat/templates/_helpers.tpl create mode 100644 charts/pgcat/templates/deployment.yaml create mode 100644 charts/pgcat/templates/ingress.yaml create mode 100644 charts/pgcat/templates/secret.yaml create mode 100644 charts/pgcat/templates/service.yaml create mode 100644 charts/pgcat/templates/serviceaccount.yaml create mode 100644 charts/pgcat/values.yaml create mode 100644 cr.yaml create mode 100644 ct.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2dd575258..7f3bf9d2d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,7 @@ updates: commit-message: prefix: "chore(deps)" open-pull-requests-limit: 10 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index 545a18f0f..e3bfca2be 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -2,7 +2,9 @@ name: Build and Push on: push: - branches: + paths: + - '!charts/**.md' + branches: - main tags: - v* diff --git a/.github/workflows/chart-lint-test.yaml b/.github/workflows/chart-lint-test.yaml new file mode 100644 index 000000000..c5a8508cd --- /dev/null +++ b/.github/workflows/chart-lint-test.yaml @@ -0,0 +1,50 @@ +name: Lint and Test Charts + +on: + pull_request: + paths: + - charts/** + - '!charts/**.md' +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3.1.0 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.8.1 + + # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and + # yamllint (https://github.com/adrienverge/yamllint) which require Python + - name: Set up Python + uses: actions/setup-python@v4.1.0 + with: + python-version: 3.7 + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.2.1 + with: + version: v3.5.1 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --config ct.yaml) + if [[ -n "$changed" ]]; then + echo "changed=true" >> $GITHUB_OUTPUT + fi + + - name: Run chart-testing (lint) + run: ct lint --config ct.yaml + + - name: Create kind cluster + uses: helm/kind-action@v1.7.0 + if: steps.list-changed.outputs.changed == 'true' + + - name: Run chart-testing (install) + run: ct install --config ct.yaml diff --git a/.github/workflows/chart-release.yaml b/.github/workflows/chart-release.yaml new file mode 100644 index 000000000..6399d4b45 --- /dev/null +++ b/.github/workflows/chart-release.yaml @@ -0,0 +1,40 @@ +name: Release Charts + +on: + push: + paths: + - charts/** + - '!**.md' + branches: + - main + +jobs: + release: + runs-on: ubuntu-latest + + permissions: + contents: write + + steps: + - name: Checkout + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 + with: + version: v3.13.0 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@be16258da8010256c6e82849661221415f031968 # v1.5.0 + with: + charts_dir: charts + config: cr.yaml + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/generate-chart-readme.yaml b/.github/workflows/generate-chart-readme.yaml new file mode 100644 index 000000000..affc40cf5 --- /dev/null +++ b/.github/workflows/generate-chart-readme.yaml @@ -0,0 +1,48 @@ +name: '[CI/CD] Update README metadata' + +on: + pull_request_target: + branches: + - main + paths: + - 'charts/*/values.yaml' +# Remove all permissions by default +permissions: {} +jobs: + update-readme-metadata: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Install readme-generator-for-helm + run: npm install -g @bitnami/readme-generator-for-helm + - name: Checkout + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + with: + path: charts + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + token: ${{ secrets.GITHUB_TOKEN }} + - name: Execute readme-generator-for-helm + env: + DIFF_URL: "${{github.event.pull_request.diff_url}}" + TEMP_FILE: "${{runner.temp}}/pr-${{github.event.number}}.diff" + run: | + # This request doesn't consume API calls. + curl -Lkso $TEMP_FILE $DIFF_URL + files_changed="$(sed -nr 's/[\-\+]{3} [ab]\/(.*)/\1/p' $TEMP_FILE | sort | uniq)" + # Adding || true to avoid "Process exited with code 1" errors + charts_dirs_changed="$(echo "$files_changed" | xargs dirname | grep -o "pgcat/[^/]*" | sort | uniq || true)" + for chart in ${charts_dirs_changed}; do + echo "Updating README.md for ${chart}" + readme-generator --values "charts/${chart}/values.yaml" --readme "charts/${chart}/README.md" --schema "/tmp/schema.json" + done + - name: Push changes + run: | + # Push all the changes + cd charts + if git status -s | grep pgcat; then + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + git add . && git commit -am "Update README.md with readme-generator-for-helm" --signoff && git push + fi diff --git a/charts/pgcat/.helmignore b/charts/pgcat/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/pgcat/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/pgcat/Chart.yaml b/charts/pgcat/Chart.yaml new file mode 100644 index 000000000..c77d29c0d --- /dev/null +++ b/charts/pgcat/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +name: pgcat +description: A Helm chart for PgCat a PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load balancing, failover and mirroring. +maintainers: + - name: Wildcard + email: support@w6d.io +appVersion: "1.1.1" +version: 0.1.0 diff --git a/charts/pgcat/templates/NOTES.txt b/charts/pgcat/templates/NOTES.txt new file mode 100644 index 000000000..552645b3f --- /dev/null +++ b/charts/pgcat/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "pgcat.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "pgcat.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "pgcat.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "pgcat.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/pgcat/templates/_config.tpl b/charts/pgcat/templates/_config.tpl new file mode 100644 index 000000000..09423a116 --- /dev/null +++ b/charts/pgcat/templates/_config.tpl @@ -0,0 +1,3 @@ +{{/* + Configuration template definition +*/}} diff --git a/charts/pgcat/templates/_helpers.tpl b/charts/pgcat/templates/_helpers.tpl new file mode 100644 index 000000000..07c2d25f4 --- /dev/null +++ b/charts/pgcat/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "pgcat.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "pgcat.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "pgcat.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "pgcat.labels" -}} +helm.sh/chart: {{ include "pgcat.chart" . }} +{{ include "pgcat.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "pgcat.selectorLabels" -}} +app.kubernetes.io/name: {{ include "pgcat.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "pgcat.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "pgcat.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/pgcat/templates/deployment.yaml b/charts/pgcat/templates/deployment.yaml new file mode 100644 index 000000000..84c57f1be --- /dev/null +++ b/charts/pgcat/templates/deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "pgcat.fullname" . }} + labels: + {{- include "pgcat.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "pgcat.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "pgcat.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "pgcat.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.containerSecurityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: pgcat + containerPort: {{ .Values.configuration.general.port }} + protocol: TCP + livenessProbe: + tcpSocket: + port: pgcat + readinessProbe: + tcpSocket: + port: pgcat + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - mountPath: /etc/pgcat + name: config + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - secret: + defaultMode: 420 + secretName: {{ include "pgcat.fullname" . }} + name: config diff --git a/charts/pgcat/templates/ingress.yaml b/charts/pgcat/templates/ingress.yaml new file mode 100644 index 000000000..1b1a5b389 --- /dev/null +++ b/charts/pgcat/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "pgcat.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "pgcat.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/pgcat/templates/secret.yaml b/charts/pgcat/templates/secret.yaml new file mode 100644 index 000000000..05711ea99 --- /dev/null +++ b/charts/pgcat/templates/secret.yaml @@ -0,0 +1,86 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "pgcat.fullname" . }} + labels: + {{- include "pgcat.labels" . | nindent 4 }} +type: Opaque +stringData: + pgcat.toml: | + [general] + host = {{ .Values.configuration.general.host | quote }} + port = {{ .Values.configuration.general.port }} + enable_prometheus_exporter = {{ .Values.configuration.general.enable_prometheus_exporter }} + prometheus_exporter_port = {{ .Values.configuration.general.prometheus_exporter_port }} + connect_timeout = {{ .Values.configuration.general.connect_timeout }} + idle_timeout = {{ .Values.configuration.general.idle_timeout | int }} + server_lifetime = {{ .Values.configuration.general.server_lifetime | int }} + idle_client_in_transaction_timeout = {{ .Values.configuration.general.idle_client_in_transaction_timeout | int }} + healthcheck_timeout = {{ .Values.configuration.general.healthcheck_timeout }} + healthcheck_delay = {{ .Values.configuration.general.healthcheck_delay }} + shutdown_timeout = {{ .Values.configuration.general.shutdown_timeout }} + ban_time = {{ .Values.configuration.general.ban_time }} + log_client_connections = {{ .Values.configuration.general.log_client_connections }} + log_client_disconnections = {{ .Values.configuration.general.log_client_disconnections }} + tcp_keepalives_idle = {{ .Values.configuration.general.tcp_keepalives_idle }} + tcp_keepalives_count = {{ .Values.configuration.general.tcp_keepalives_count }} + tcp_keepalives_interval = {{ .Values.configuration.general.tcp_keepalives_interval }} + {{- if and (ne .Values.configuration.general.tls_certificate "-") (ne .Values.configuration.general.tls_private_key "-") }} + tls_certificate = "{{ .Values.configuration.general.tls_certificate }}" + tls_private_key = "{{ .Values.configuration.general.tls_private_key }}" + {{- end }} + admin_username = {{ .Values.configuration.general.admin_username | quote }} + admin_password = {{ .Values.configuration.general.admin_password | quote }} + {{- if and .Values.configuration.general.auth_query_user .Values.configuration.general.auth_query_password .Values.configuration.general.auth_query }} + auth_query = {{ .Values.configuration.general.auth_query | quote }} + auth_query_user = {{ .Values.configuration.general.auth_query_user | quote }} + auth_query_password = {{ .Values.configuration.general.auth_query_password | quote }} + {{- end }} + + {{- range $pool := .Values.configuration.pools }} + + ## + ## pool for {{ $pool.name }} + ## + [pools.{{ $pool.name | quote }}] + pool_mode = {{ default "transaction" $pool.pool_mode | quote }} + load_balancing_mode = {{ default "random" $pool.load_balancing_mode | quote }} + default_role = {{ default "any" $pool.default_role | quote }} + prepared_statements_cache_size = {{ default 500 $pool.prepared_statements_cache_size }} + query_parser_enabled = {{ default true $pool.query_parser_enabled }} + query_parser_read_write_splitting = {{ default true $pool.query_parser_read_write_splitting }} + primary_reads_enabled = {{ default true $pool.primary_reads_enabled }} + sharding_function = {{ default "pg_bigint_hash" $pool.sharding_function | quote }} + + {{- range $index, $user := $pool.users }} + + ## pool {{ $pool.name }} user {{ $user.username | quote }} + ## + [pools.{{ $pool.name | quote }}.users.{{ $index }}] + username = {{ $user.username | quote }} + password = {{ $user.password | quote }} + pool_size = {{ $user.pool_size }} + statement_timeout = {{ $user.statement_timeout }} + min_pool_size = 3 + server_lifetime = 60000 + {{- if and $user.server_username $user.server_password }} + server_username = {{ $user.server_username | quote }} + server_password = {{ $user.server_password | quote }} + {{- end }} + {{- end }} + + {{- range $index, $shard := $pool.shards }} + + ## pool {{ $pool.name }} database {{ $shard.database }} + ## + [pools.{{ $pool.name | quote }}.shards.{{ $index }}] + {{- if gt (len $shard.servers) 0}} + servers = [ + {{- range $server := $shard.servers }} + [ {{ $server.host | quote }}, {{ $server.port }}, {{ $server.role | quote }} ], + {{- end }} + ] + {{- end }} + database = {{ $shard.database | quote }} + {{- end }} + {{- end }} diff --git a/charts/pgcat/templates/service.yaml b/charts/pgcat/templates/service.yaml new file mode 100644 index 000000000..56c4be716 --- /dev/null +++ b/charts/pgcat/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "pgcat.fullname" . }} + labels: + {{- include "pgcat.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: pgcat + protocol: TCP + name: pgcat + selector: + {{- include "pgcat.selectorLabels" . | nindent 4 }} diff --git a/charts/pgcat/templates/serviceaccount.yaml b/charts/pgcat/templates/serviceaccount.yaml new file mode 100644 index 000000000..04ea6161b --- /dev/null +++ b/charts/pgcat/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "pgcat.serviceAccountName" . }} + labels: + {{- include "pgcat.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/pgcat/values.yaml b/charts/pgcat/values.yaml new file mode 100644 index 000000000..86be4104c --- /dev/null +++ b/charts/pgcat/values.yaml @@ -0,0 +1,361 @@ +## String to partially override aspnet-core.fullname template (will maintain the release name) +## @param nameOverride String to partially override common.names.fullname +## +nameOverride: "" + +## String to fully override aspnet-core.fullname template +## @param fullnameOverride String to fully override common.names.fullname +## +fullnameOverride: "" + +## Number of PgCat replicas to deploy +## @param replicaCount Number of PgCat replicas to deploy +replicaCount: 1 + +## Bitnami PgCat image version +## ref: https://hub.docker.com/r/bitnami/kubewatch/tags/ +## +## @param image.registry PgCat image registry +## @param image.repository PgCat image name +## @param image.tag PgCat image tag +## @param image.pullPolicy PgCat image tag +## @param image.pullSecrets Specify docker-registry secret names as an array +image: + repository: ghcr.io/postgresml/pgcat + # Overrides the image tag whose default is the chart appVersion. + tag: "main" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + +## Specifies whether a ServiceAccount should be created +## +## @param serviceAccount.create Enable the creation of a ServiceAccount for PgCat pods +## @param serviceAccount.name Name of the created ServiceAccount +## +serviceAccount: + ## Specifies whether a service account should be created + create: true + ## Annotations to add to the service account + annotations: {} + ## The name of the service account to use. + ## If not set and create is true, a name is generated using the fullname template + name: "" + +## Annotations for server pods. +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +## @param podAnnotations Annotations for PgCat pods +## +podAnnotations: {} + +## PgCat containers' SecurityContext +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +## +## @param podSecurityContext.enabled Enabled PgCat pods' Security Context +## @param podSecurityContext.fsGroup Set PgCat pod's Security Context fsGroup +## +podSecurityContext: {} + # fsGroup: 2000 + +## PgCat pods' Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +## +## @param containerSecurityContext.enabled Enabled PgCat containers' Security Context +## @param containerSecurityContext.runAsUser Set PgCat container's Security Context runAsUser +## @param containerSecurityContext.runAsNonRoot Set PgCat container's Security Context runAsNonRoot +## +containerSecurityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +## PgCat service +## +## @param service.type PgCat service type +## @param service.port PgCat service port +service: + type: ClusterIP + port: 6432 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +## PgCat resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +## @skip resources Optional description +## @disabled-param resources.limits The resources limits for the PgCat container +## @disabled-param resources.requests The requested resources for the PgCat container +## +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 100m + # memory: 128Mi + requests: {} + # cpu: 100m + # memory: 128Mi + +## Node labels for pod assignment. Evaluated as a template. +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +## @param nodeSelector Node labels for pod assignment +## +nodeSelector: {} + +## Tolerations for pod assignment. Evaluated as a template. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +## @param tolerations Tolerations for pod assignment +## +tolerations: [] + +## Affinity for pod assignment. Evaluated as a template. +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## Note: podAffinityPreset, podAntiAffinityPreset, and nodeAffinityPreset will be ignored when it's set +## +## @param affinity Affinity for pod assignment +## +affinity: {} + +## PgCat configuration +## @param configuration [object] +configuration: + ## General pooler settings + ## @param [object] + general: + ## @param configuration.general.host What IP to run on, 0.0.0.0 means accessible from everywhere. + host: "0.0.0.0" + + ## @param configuration.general.port Port to run on, same as PgBouncer used in this example. + port: 6432 + + ## @param configuration.general.enable_prometheus_exporter Whether to enable prometheus exporter or not. + enable_prometheus_exporter: false + + ## @param configuration.general.prometheus_exporter_port Port at which prometheus exporter listens on. + prometheus_exporter_port: 9930 + + # @param configuration.general.connect_timeout How long to wait before aborting a server connection (ms). + connect_timeout: 5000 + + # How long an idle connection with a server is left open (ms). + idle_timeout: 30000 # milliseconds + + # Max connection lifetime before it's closed, even if actively used. + server_lifetime: 86400000 # 24 hours + + # How long a client is allowed to be idle while in a transaction (ms). + idle_client_in_transaction_timeout: 0 # milliseconds + + # @param configuration.general.healthcheck_timeout How much time to give `SELECT 1` health check query to return with a result (ms). + healthcheck_timeout: 1000 + + # @param configuration.general.healthcheck_delay How long to keep connection available for immediate re-use, without running a healthcheck query on it + healthcheck_delay: 30000 + + # @param configuration.general.shutdown_timeout How much time to give clients during shutdown before forcibly killing client connections (ms). + shutdown_timeout: 60000 + + # @param configuration.general.ban_time For how long to ban a server if it fails a health check (seconds). + ban_time: 60 # seconds + + # @param configuration.general.log_client_connections If we should log client connections + log_client_connections: false + + # @param configuration.general.log_client_disconnections If we should log client disconnections + log_client_disconnections: false + + # TLS + # tls_certificate: "server.cert" + # tls_private_key: "server.key" + tls_certificate: "-" + tls_private_key: "-" + + # Credentials to access the virtual administrative database (pgbouncer or pgcat) + # Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc.. + admin_username: "postgres" + admin_password: "postgres" + + # Query to be sent to servers to obtain the hash used for md5 authentication. The connection will be + # established using the database configured in the pool. This parameter is inherited by every pool and + # can be redefined in pool configuration. + auth_query: null + + # User to be used for connecting to servers to obtain the hash used for md5 authentication by sending + # the query specified in auth_query_user. The connection will be established using the database configured + # in the pool. This parameter is inherited by every pool and can be redefined in pool configuration. + # + # @param configuration.general.auth_query_user + auth_query_user: null + + # Password to be used for connecting to servers to obtain the hash used for md5 authentication by sending + # the query specified in auth_query_user. The connection will be established using the database configured + # in the pool. This parameter is inherited by every pool and can be redefined in pool configuration. + # + # @param configuration.general.auth_query_password + auth_query_password: null + + # Number of seconds of connection idleness to wait before sending a keepalive packet to the server. + tcp_keepalives_idle: 5 + + # Number of unacknowledged keepalive packets allowed before giving up and closing the connection. + tcp_keepalives_count: 5 + + # Number of seconds between keepalive packets. + tcp_keepalives_interval: 5 + + ## pool + ## configs are structured as pool. + ## the pool_name is what clients use as database name when connecting + ## For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded" + ## @param [object] + pools: [] + # - ## default values + # ## + # ## + # ## + # name: "db" + + # ## Pool mode (see PgBouncer docs for more). + # ## session: one server connection per connected client + # ## transaction: one server connection per client transaction + # ## @param configuration.poolsPostgres.pool_mode + # pool_mode: "transaction" + + # ## Load balancing mode + # ## `random` selects the server at random + # ## `loc` selects the server with the least outstanding busy connections + # ## + # ## @param configuration.poolsPostgres.load_balancing_mode + # load_balancing_mode: "random" + + # ## Prepared statements cache size. + # ## TODO: update documentation + # ## + # ## @param configuration.poolsPostgres.prepared_statements_cache_size + # prepared_statements_cache_size: 500 + + # ## If the client doesn't specify, route traffic to + # ## this role by default. + # ## + # ## any: round-robin between primary and replicas, + # ## replica: round-robin between replicas only without touching the primary, + # ## primary: all queries go to the primary unless otherwise specified. + # ## @param configuration.poolsPostgres.default_role + # default_role: "any" + + # ## Query parser. If enabled, we'll attempt to parse + # ## every incoming query to determine if it's a read or a write. + # ## If it's a read query, we'll direct it to a replica. Otherwise, if it's a write, + # ## we'll direct it to the primary. + # ## @param configuration.poolsPostgres.query_parser_enabled + # query_parser_enabled: true + + # ## If the query parser is enabled and this setting is enabled, we'll attempt to + # ## infer the role from the query itself. + # ## @param configuration.poolsPostgres.query_parser_read_write_splitting + # query_parser_read_write_splitting: true + + # ## If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for + # ## load balancing of read queries. Otherwise, the primary will only be used for write + # ## queries. The primary can always be explicitly selected with our custom protocol. + # ## @param configuration.poolsPostgres.primary_reads_enabled + # primary_reads_enabled: true + + # ## So what if you wanted to implement a different hashing function, + # ## or you've already built one and you want this pooler to use it? + # ## + # ## Current options: + # ## + # ## pg_bigint_hash: PARTITION BY HASH (Postgres hashing function) + # ## sha1: A hashing function based on SHA1 + # ## + # ## @param configuration.poolsPostgres.sharding_function + # sharding_function: "pg_bigint_hash" + + # ## Credentials for users that may connect to this cluster + # ## @param users [array] + # ## @param users[0].username Name of the env var (required) + # ## @param users[0].password Value for the env var (required) + # ## @param users[0].pool_size Maximum number of server connections that can be established for this user + # ## @param users[0].statement_timeout Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. + # users: [] + # # - username: "user" + # # password: "pass" + # # + # # # The maximum number of connection from a single Pgcat process to any database in the cluster + # # # is the sum of pool_size across all users. + # # pool_size: 9 + # # + # # # Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. + # # statement_timeout: 0 + # # + # # # PostgreSQL username used to connect to the server. + # # server_username: "postgres + # # + # # # PostgreSQL password used to connect to the server. + # # server_password: "postgres + + # ## @param shards [array] + # ## @param shards[0].server[0].host Host for this shard + # ## @param shards[0].server[0].port Port for this shard + # ## @param shards[0].server[0].role Role for this shard + # shards: [] + # # [ host, port, role ] + # # - servers: + # # - host: "postgres" + # # port: 5432 + # # role: "primary" + # # - host: "postgres" + # # port: 5432 + # # role: "replica" + # # database: "postgres" + # # # [ host, port, role ] + # # - servers: + # # - host: "postgres" + # # port: 5432 + # # role: "primary" + # # - host: "postgres" + # # port: 5432 + # # role: "replica" + # # database: "postgres" + # # # [ host, port, role ] + # # - servers: + # # - host: "postgres" + # # port: 5432 + # # role: "primary" + # # - host: "postgres" + # # port: 5432 + # # role: "replica" + # # database: "postgres" diff --git a/cr.yaml b/cr.yaml new file mode 100644 index 000000000..0e3201dd6 --- /dev/null +++ b/cr.yaml @@ -0,0 +1 @@ +sign: false diff --git a/ct.yaml b/ct.yaml new file mode 100644 index 000000000..fe05590af --- /dev/null +++ b/ct.yaml @@ -0,0 +1,5 @@ +remote: origin +target-branch: main +chart-dirs: + - charts + From 882d870a9c026e23066d01619412627b4f9b3c87 Mon Sep 17 00:00:00 2001 From: Toby Hede Date: Sat, 13 Apr 2024 15:12:37 +1000 Subject: [PATCH 495/551] Update sqlparser to 0.41 (#666) --- Cargo.lock | 10 +++++----- Cargo.toml | 9 ++++++--- src/query_router.rs | 7 ++++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28f20a9da..6699ce614 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1510,9 +1510,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.34.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3706eefb17039056234df6b566b0014f303f867f2656108334a55b8096f59" +checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" dependencies = [ "log", "sqlparser_derive", @@ -1520,13 +1520,13 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.1.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55fe75cb4a364c7f7ae06c7dbbc8d84bddd85d6cdf9975963c3935bc1991761e" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.26", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f451ffc5b..f408ba4e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = {version = "0.34", features = ["visitor"] } +sqlparser = { version = "0.41", features = ["visitor"] } log = "0.4" arc-swap = "1" parking_lot = "0.12.1" @@ -47,9 +47,12 @@ serde_json = "1" itertools = "0.10" clap = { version = "4.3.1", features = ["derive", "env"] } tracing = "0.1.37" -tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter", "std"]} +tracing-subscriber = { version = "0.3.17", features = [ + "json", + "env-filter", + "std", +] } lru = "0.12.0" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" - diff --git a/src/query_router.rs b/src/query_router.rs index 541883f22..7acd6841b 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -499,6 +499,7 @@ impl QueryRouter { table: _, on: _, returning: _, + ignore: _, } => { // Not supported in postgres. assert!(or.is_none()); @@ -506,7 +507,9 @@ impl QueryRouter { assert!(after_columns.is_empty()); Self::process_table(table_name, &mut table_names); - Self::process_query(source, &mut exprs, &mut table_names, &Some(columns)); + if let Some(source) = source { + Self::process_query(source, &mut exprs, &mut table_names, &Some(columns)); + } } Delete { tables, @@ -514,6 +517,8 @@ impl QueryRouter { using, selection, returning: _, + order_by: _, + limit: _, } => { if let Some(expr) = selection { exprs.push(expr.clone()); From 2e0b071cde78714950d24836bf6ad03a9d704b0d Mon Sep 17 00:00:00 2001 From: Horacio <151622129+horacio-qogita@users.noreply.github.com> Date: Fri, 19 Apr 2024 18:12:57 +0200 Subject: [PATCH 496/551] Use rust:bullseye as base image (#725) Use rust:bullseye base image With the original rust:1.70-bullseye image, the container cannot be built: 17.06 Installing /usr/local/cargo/bin/rustfilt 17.06 Installed package `rustfilt v0.2.1` (executable `rustfilt`) 17.06 error: failed to compile `cargo-binutils v0.3.6`, intermediate artifacts can be found at `/tmp/cargo-installrc6mPb` 17.06 17.06 Caused by: 17.06 package `cargo-platform v0.1.8` cannot be built because it requires rustc 1.73 or newer, while the currently active rustc version is 1.70.0 17.06 Try re-running cargo install with `--locked` 17.06 Summary Successfully installed rustfilt! Failed to install cargo-binutils (see error(s) above). 17.06 error: some crates failed to install This is the same base image used on tests/docker/Dockerfile --- dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/Dockerfile b/dev/Dockerfile index c3761b75c..dc0386733 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.70-bullseye +FROM rust:bullseye # Dependencies COPY --from=sclevine/yj /bin/yj /bin/yj From 9b06c5b77193c1fc5558503dce9047a56d8eef62 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 8 May 2024 12:18:27 -0500 Subject: [PATCH 497/551] Report checkout error when all servers are down (#736) We shouldn't report checkout_success when we are going to return Error. --- src/pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pool.rs b/src/pool.rs index c03aaf4c9..7915a0a41 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -813,7 +813,7 @@ impl ConnectionPool { } } - client_stats.checkout_success(); + client_stats.checkout_error(); Err(Error::AllServersDown) } From d8c3a86f89a6df0154e8935d248d9b53b853e6b8 Mon Sep 17 00:00:00 2001 From: Andrey Stikheev Date: Mon, 27 May 2024 00:47:21 +0300 Subject: [PATCH 498/551] Add TCP_NODELAY option to improve performance for large response queries (#749) This commit adds the TCP_NODELAY option to the socket configuration in `configure_socket` function. Without this option, we observed significant performance issues when executing SELECT queries with large responses. Before the fix: postgres=> SELECT repeat('a', 1); SELECT repeat('a', 8153); Time: 1.368 ms Time: 41.364 ms After the fix: postgres=> SELECT repeat('a', 1); SELECT repeat('a', 8153); Time: 1.332 ms Time: 1.528 ms By setting TCP_NODELAY, we eliminate the Nagle's algorithm delay, which results in a substantial improvement in response times for large queries. This problem was discussed in https://github.com/postgresml/pgcat/issues/616. --- src/messages.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/messages.rs b/src/messages.rs index 4390d9f9b..6a114e139 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -733,6 +733,10 @@ pub fn configure_socket(stream: &TcpStream) { } Err(err) => error!("Could not configure socket: {}", err), } + match sock_ref.set_nodelay(true) { + Ok(_) => (), + Err(err) => error!("Could not configure TCP_NODELAY for socket: {}", err), + } } pub trait BytesMutReader { From 4792b43c2fdb3b5524d6d07e753bb0b3d6542186 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 21 Jun 2024 01:23:41 -0500 Subject: [PATCH 499/551] Add STOPSIGNAL to Dockerfile (#758) --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index be3df5365..ddfe99520 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,3 +19,4 @@ COPY --from=builder /app/pgcat.toml /etc/pgcat/pgcat.toml WORKDIR /etc/pgcat ENV RUST_LOG=info CMD ["pgcat"] +STOPSIGNAL SIGINT From ede536dbd1ffde94d3bcd354cc8931e44c95a30a Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 24 Jun 2024 10:51:38 -0500 Subject: [PATCH 500/551] update rust version in docker image (#762) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ddfe99520..8a7edeae9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1-slim-bookworm AS builder +FROM rust:1.79.0-slim-bookworm AS builder RUN apt-get update && \ apt-get install -y build-essential From 6f53b982a2d1ce2a9ec2158ba859c62d06097654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3=20=D0=94=D1=83=D0=BB=D0=B5=D1=86?= =?UTF-8?q?=D0=BA=D0=B8=D0=B9?= Date: Mon, 24 Jun 2024 17:57:58 +0200 Subject: [PATCH 501/551] Add ExecReload option to pgcat.service for configuration reloads (#760) --- pgcat.service | 1 + 1 file changed, 1 insertion(+) diff --git a/pgcat.service b/pgcat.service index 4da4c7c00..3b27cb339 100644 --- a/pgcat.service +++ b/pgcat.service @@ -11,6 +11,7 @@ RestartSec=1 Environment=RUST_LOG=info LimitNOFILE=65536 ExecStart=/usr/bin/pgcat /etc/pgcat.toml +ExecReload=/bin/kill -SIGHUP $MAINPID [Install] WantedBy=multi-user.target From 4fa82044ed4940a59ba5184ea49b54eff6146280 Mon Sep 17 00:00:00 2001 From: brandonpike Date: Mon, 15 Jul 2024 22:30:26 -0500 Subject: [PATCH 502/551] Fix lint warnings for rust-1.79 (#769) 2 things that are recommended by rust-lang - implementing `std::fmt::Display` rather than ToString (1) and using clone_from (2). [1] https://rust-lang.github.io/rust-clippy/master/index.html#/to_string_trait_impl [2] https://rust-lang.github.io/rust-clippy/master/index.html#assigning_clones Signed-off-by: Brandon Pike --- src/config.rs | 43 +++++++++++++++++++++++-------------------- src/sharding.rs | 10 +++++----- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/config.rs b/src/config.rs index ef7952f29..c7aaf4c35 100644 --- a/src/config.rs +++ b/src/config.rs @@ -38,12 +38,12 @@ pub enum Role { Mirror, } -impl ToString for Role { - fn to_string(&self) -> String { - match *self { - Role::Primary => "primary".to_string(), - Role::Replica => "replica".to_string(), - Role::Mirror => "mirror".to_string(), +impl std::fmt::Display for Role { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Role::Primary => write!(f, "primary"), + Role::Replica => write!(f, "replica"), + Role::Mirror => write!(f, "mirror"), } } } @@ -476,11 +476,11 @@ pub enum PoolMode { Session, } -impl ToString for PoolMode { - fn to_string(&self) -> String { - match *self { - PoolMode::Transaction => "transaction".to_string(), - PoolMode::Session => "session".to_string(), +impl std::fmt::Display for PoolMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PoolMode::Transaction => write!(f, "transaction"), + PoolMode::Session => write!(f, "session"), } } } @@ -493,12 +493,13 @@ pub enum LoadBalancingMode { #[serde(alias = "loc", alias = "LOC", alias = "least_outstanding_connections")] LeastOutstandingConnections, } -impl ToString for LoadBalancingMode { - fn to_string(&self) -> String { - match *self { - LoadBalancingMode::Random => "random".to_string(), + +impl std::fmt::Display for LoadBalancingMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + LoadBalancingMode::Random => write!(f, "random"), LoadBalancingMode::LeastOutstandingConnections => { - "least_outstanding_connections".to_string() + write!(f, "least_outstanding_connections") } } } @@ -999,15 +1000,17 @@ impl Config { pub fn fill_up_auth_query_config(&mut self) { for (_name, pool) in self.pools.iter_mut() { if pool.auth_query.is_none() { - pool.auth_query = self.general.auth_query.clone(); + pool.auth_query.clone_from(&self.general.auth_query); } if pool.auth_query_user.is_none() { - pool.auth_query_user = self.general.auth_query_user.clone(); + pool.auth_query_user + .clone_from(&self.general.auth_query_user); } if pool.auth_query_password.is_none() { - pool.auth_query_password = self.general.auth_query_password.clone(); + pool.auth_query_password + .clone_from(&self.general.auth_query_password); } } } @@ -1155,7 +1158,7 @@ impl Config { "Default max server lifetime: {}ms", self.general.server_lifetime ); - info!("Sever round robin: {}", self.general.server_round_robin); + info!("Server round robin: {}", self.general.server_round_robin); match self.general.tls_certificate.clone() { Some(tls_certificate) => { info!("TLS certificate: {}", tls_certificate); diff --git a/src/sharding.rs b/src/sharding.rs index a7a9df130..990f967b0 100644 --- a/src/sharding.rs +++ b/src/sharding.rs @@ -14,11 +14,11 @@ pub enum ShardingFunction { Sha1, } -impl ToString for ShardingFunction { - fn to_string(&self) -> String { - match *self { - ShardingFunction::PgBigintHash => "pg_bigint_hash".to_string(), - ShardingFunction::Sha1 => "sha1".to_string(), +impl std::fmt::Display for ShardingFunction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ShardingFunction::PgBigintHash => write!(f, "pg_bigint_hash"), + ShardingFunction::Sha1 => write!(f, "sha1"), } } } From 61c1aa7bbd2d895c2c4cccdbae2b74f71ac3b189 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Wed, 28 Aug 2024 21:29:40 -0500 Subject: [PATCH 503/551] Fix a Panic in admin commands (#779) We have a panic when we send SHOW or ;;;;;;;;;;;;;;;;; to admin database. This PR fixes these panics and adds a couple of tests --- src/admin.rs | 14 ++++++++++++-- tests/ruby/admin_spec.rb | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/admin.rs b/src/admin.rs index 22bbb0af1..f08ef2e1c 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -55,7 +55,12 @@ where let query_parts: Vec<&str> = query.trim_end_matches(';').split_whitespace().collect(); - match query_parts[0].to_ascii_uppercase().as_str() { + match query_parts + .first() + .unwrap_or(&"") + .to_ascii_uppercase() + .as_str() + { "BAN" => { trace!("BAN"); ban(stream, query_parts).await @@ -84,7 +89,12 @@ where trace!("SHUTDOWN"); shutdown(stream).await } - "SHOW" => match query_parts[1].to_ascii_uppercase().as_str() { + "SHOW" => match query_parts + .get(1) + .unwrap_or(&"") + .to_ascii_uppercase() + .as_str() + { "HELP" => { trace!("SHOW HELP"); show_help(stream).await diff --git a/tests/ruby/admin_spec.rb b/tests/ruby/admin_spec.rb index abaa5ff4d..f9613d96e 100644 --- a/tests/ruby/admin_spec.rb +++ b/tests/ruby/admin_spec.rb @@ -91,6 +91,27 @@ end end + [ + "SHOW ME THE MONEY", + "SHOW ME THE WAY", + "SHOW UP", + "SHOWTIME", + "HAMMER TIME", + "SHOWN TO BE TRUE", + "SHOW ", + "SHOW ", + "SHOW 1", + ";;;;;" + ].each do |cmd| + describe "Bad command #{cmd}" do + it "does not panic and responds with PG::SystemError" do + admin_conn = PG::connect(processes.pgcat.admin_connection_string) + expect { admin_conn.async_exec(cmd) }.to raise_error(PG::SystemError).with_message(/Unsupported/) + admin_conn.close + end + end + end + describe "PAUSE" do it "pauses all pools" do admin_conn = PG::connect(processes.pgcat.admin_connection_string) From 0d96ba521f965b5b0d3ef1fc6725a7a60ad65af1 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 29 Aug 2024 08:37:13 -0500 Subject: [PATCH 504/551] Fix CI image build step (#780) The docker CI build image is failing due to this error 249.5 Finished release [optimized] target(s) in 2m 49s 249.5 Installing /home/circleci/.cargo/bin/rustfilt 249.5 Installed package `rustfilt v0.2.1` (executable `rustfilt`) 249.5 error: failed to compile `cargo-binutils v0.3.6`, intermediate artifacts can be found at `/tmp/cargo-installrWENQG` 249.5 249.5 Caused by: 249.5 package `cargo-platform v0.1.8` cannot be built because it requires rustc 1.73 or newer, while the currently active rustc version is 1.67.1 249.5 Try re-running cargo install with `--locked` 249.5 Summary Successfully installed rustfilt! Failed to install cargo-binutils (see error(s) above). 249.5 error: some crates failed to install So I am bumping the version up --- Dockerfile.ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 57d28b87e..565418d41 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM cimg/rust:1.67.1 +FROM cimg/rust:1.79.0 COPY --from=sclevine/yj /bin/yj /bin/yj RUN /bin/yj -h RUN sudo apt-get update && \ From 0dde5eab18d228849d9a0ce988cd837eaef2bb80 Mon Sep 17 00:00:00 2001 From: Saraj Munjal Date: Thu, 29 Aug 2024 07:47:58 -0700 Subject: [PATCH 505/551] Bump the hyper crate to v1.4.1 and rework prometheus server handling (#778) Bump hyper to v1.4.1 and rework prometheus server handling --- Cargo.lock | 92 ++++++++++++++++++++++++++--------------------- Cargo.toml | 4 ++- src/prometheus.rs | 46 +++++++++++++++++++----- 3 files changed, 92 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6699ce614..e0dff6e10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,6 +146,12 @@ dependencies = [ "syn 2.0.26", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "atomic_enum" version = "0.2.0" @@ -542,29 +548,23 @@ checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" [[package]] name = "h2" -version = "0.3.20" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http", - "indexmap 1.9.3", + "indexmap", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.0" @@ -609,9 +609,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -620,12 +620,24 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", "pin-project-lite", ] @@ -643,13 +655,12 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.27" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", "h2", "http", @@ -658,13 +669,26 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] +[[package]] +name = "hyper-util" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -709,16 +733,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.0.0" @@ -726,7 +740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown", ] [[package]] @@ -848,7 +862,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" dependencies = [ - "hashbrown 0.14.0", + "hashbrown", ] [[package]] @@ -1034,7 +1048,9 @@ dependencies = [ "fallible-iterator", "futures", "hmac", + "http-body-util", "hyper", + "hyper-util", "itertools", "jemallocator", "log", @@ -1478,9 +1494,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" @@ -1741,19 +1757,13 @@ version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap 2.0.0", + "indexmap", "serde", "serde_spanned", "toml_datetime", "winnow", ] -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - [[package]] name = "tracing" version = "0.1.37" diff --git a/Cargo.toml b/Cargo.toml index f408ba4e2..390d9d5dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,9 @@ base64 = "0.21" stringprep = "0.1" tokio-rustls = "0.24" rustls-pemfile = "1" -hyper = { version = "0.14", features = ["full"] } +http-body-util = "0.1.2" +hyper = { version = "1.4.1", features = ["full"] } +hyper-util = { version = "0.1.7", features = ["tokio"] } phf = { version = "0.11.1", features = ["macros"] } exitcode = "1.1.2" futures = "0.3" diff --git a/src/prometheus.rs b/src/prometheus.rs index 7e264dcad..d8e23464e 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -1,5 +1,11 @@ -use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Method, Request, Response, Server, StatusCode}; +use http_body_util::Full; +use hyper::body; +use hyper::body::Bytes; + +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Method, Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; use log::{debug, error, info}; use phf::phf_map; use std::collections::HashMap; @@ -7,6 +13,7 @@ use std::fmt; use std::net::SocketAddr; use std::sync::atomic::Ordering; use std::sync::Arc; +use tokio::net::TcpListener; use crate::config::Address; use crate::pool::{get_all_pools, PoolIdentifier}; @@ -243,7 +250,9 @@ impl PrometheusMetric { } } -async fn prometheus_stats(request: Request) -> Result, hyper::http::Error> { +async fn prometheus_stats( + request: Request, +) -> Result>, hyper::http::Error> { match (request.method(), request.uri().path()) { (&Method::GET, "/metrics") => { let mut lines = Vec::new(); @@ -374,14 +383,35 @@ fn push_server_stats(lines: &mut Vec) { } pub async fn start_metric_server(http_addr: SocketAddr) { - let http_service_factory = - make_service_fn(|_conn| async { Ok::<_, hyper::Error>(service_fn(prometheus_stats)) }); - let server = Server::bind(&http_addr).serve(http_service_factory); + let listener = TcpListener::bind(http_addr); + let listener = match listener.await { + Ok(listener) => listener, + Err(e) => { + error!("Failed to bind prometheus server to HTTP address: {}.", e); + return; + } + }; info!( "Exposing prometheus metrics on http://{}/metrics.", http_addr ); - if let Err(e) = server.await { - error!("Failed to run HTTP server: {}.", e); + loop { + let stream = match listener.accept().await { + Ok((stream, _)) => stream, + Err(e) => { + error!("Error accepting connection: {}", e); + continue; + } + }; + let io = TokioIo::new(stream); + + tokio::task::spawn(async move { + if let Err(err) = http1::Builder::new() + .serve_connection(io, service_fn(prometheus_stats)) + .await + { + eprintln!("Error serving HTTP connection for metrics: {:?}", err); + } + }); } } From 866222d3cc856acc704b0f4f017eb25d37f745a6 Mon Sep 17 00:00:00 2001 From: KwongTN <5886584+kwongtn@users.noreply.github.com> Date: Fri, 30 Aug 2024 02:50:38 +0800 Subject: [PATCH 506/551] Add linux/arm64 docker image build support (#774) --- .github/workflows/build-and-push.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml index e3bfca2be..a2f1c755d 100644 --- a/.github/workflows/build-and-push.yaml +++ b/.github/workflows/build-and-push.yaml @@ -23,14 +23,17 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Determine tags id: metadata - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.registry }}/${{ env.image-name }} tags: | @@ -42,15 +45,18 @@ jobs: type=raw,value=latest,enable={{ is_default_branch }} - name: Log in to the Container registry - uses: docker/login-action@v2.1.0 + uses: docker/login-action@v3 with: registry: ${{ env.registry }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push ${{ env.image-name }} - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: + context: . + platforms: linux/amd64,linux/arm64 + provenance: false push: true tags: ${{ steps.metadata.outputs.tags }} labels: ${{ steps.metadata.outputs.labels }} From 1d59e5dabc54864ddd82e632dc7d52f8ba3c4e89 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 30 Aug 2024 04:26:36 -0500 Subject: [PATCH 507/551] QueryRouter: route to primary when locks exists (select for update) (#782) Authored-by: Javier Goday --- src/query_router.rs | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/query_router.rs b/src/query_router.rs index 7acd6841b..da67e96a6 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -427,8 +427,12 @@ impl QueryRouter { None => (), }; - // If we already visited a write statement, we should be going to the primary. - if !visited_write_statement { + let has_locks = !query.locks.is_empty(); + + if has_locks { + self.active_role = Some(Role::Primary); + } else if !visited_write_statement { + // If we already visited a write statement, we should be going to the primary. self.active_role = match self.primary_reads_enabled() { false => Some(Role::Replica), // If primary should not be receiving reads, use a replica. true => None, // Any server role is fine in this case. @@ -1158,6 +1162,29 @@ mod test { } } + #[test] + fn test_select_for_update() { + QueryRouter::setup(); + let mut qr = QueryRouter::new(); + qr.pool_settings.query_parser_read_write_splitting = true; + + let queries_in_primary_role = vec![ + simple_query("BEGIN"), // Transaction start + simple_query("SELECT * FROM items WHERE id = 5 FOR UPDATE"), + simple_query("UPDATE items SET name = 'pumpkin' WHERE id = 5"), + ]; + + for query in queries_in_primary_role { + assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); + assert_eq!(qr.role(), Some(Role::Primary)); + } + + // query without lock do not change role + let query = simple_query("SELECT * FROM items WHERE id = 5"); + assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); + assert_eq!(qr.role(), None); + } + #[test] fn test_infer_primary_reads_enabled() { QueryRouter::setup(); From 4beecaca674d250ed482d29df230d32fde45a8a7 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 30 Aug 2024 08:30:16 -0500 Subject: [PATCH 508/551] Cut 1.2.0 release (#783) --- Cargo.lock | 2 +- Cargo.toml | 2 +- charts/pgcat/Chart.yaml | 4 ++-- charts/pgcat/values.yaml | 16 ++++++++++++---- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0dff6e10..e2f898535 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1034,7 +1034,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pgcat" -version = "1.1.2-dev4" +version = "1.2.0" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 390d9d5dc..f75e91858 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.1.2-dev4" +version = "1.2.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/charts/pgcat/Chart.yaml b/charts/pgcat/Chart.yaml index c77d29c0d..e05b239f9 100644 --- a/charts/pgcat/Chart.yaml +++ b/charts/pgcat/Chart.yaml @@ -4,5 +4,5 @@ description: A Helm chart for PgCat a PostgreSQL pooler and proxy (like PgBounce maintainers: - name: Wildcard email: support@w6d.io -appVersion: "1.1.1" -version: 0.1.0 +appVersion: "1.2.0" +version: 0.2.0 diff --git a/charts/pgcat/values.yaml b/charts/pgcat/values.yaml index 86be4104c..e87c576b7 100644 --- a/charts/pgcat/values.yaml +++ b/charts/pgcat/values.yaml @@ -170,13 +170,13 @@ configuration: connect_timeout: 5000 # How long an idle connection with a server is left open (ms). - idle_timeout: 30000 # milliseconds + idle_timeout: 30000 # milliseconds # Max connection lifetime before it's closed, even if actively used. - server_lifetime: 86400000 # 24 hours + server_lifetime: 86400000 # 24 hours # How long a client is allowed to be idle while in a transaction (ms). - idle_client_in_transaction_timeout: 0 # milliseconds + idle_client_in_transaction_timeout: 0 # milliseconds # @param configuration.general.healthcheck_timeout How much time to give `SELECT 1` health check query to return with a result (ms). healthcheck_timeout: 1000 @@ -240,7 +240,15 @@ configuration: ## the pool_name is what clients use as database name when connecting ## For the example below a client can connect using "postgres://sharding_user:sharding_user@pgcat_host:pgcat_port/sharded" ## @param [object] - pools: [] + pools: + [{ + name: "simple", pool_mode: "transaction", + users: [{username: "user", password: "pass", pool_size: 5, statement_timeout: 0}], + shards: [{ + servers: [{host: "postgres", port: 5432, role: "primary"}], + database: "postgres" + }] + }] # - ## default values # ## # ## From 08538c81fddaee14b8a9036bfd710caf1ca197d7 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sat, 31 Aug 2024 08:18:57 -0500 Subject: [PATCH 509/551] Improve Prometheus stats + Add Grafana dashboard (#785) We were missing some labels on metrics generated by the Prometheus exporter so I fixed that. There are still some gaps that I want to address with respect to the metrics we track but this seems like a good start. I also created a Grafana Dashboard and exported it to JSON. It is designed with the same metric names the Prometheus exporter uses. --- README.md | 2 + grafana_dashboard.json | 2124 ++++++++++++++++++++++++++++++++++++++++ src/prometheus.rs | 111 ++- 3 files changed, 2210 insertions(+), 27 deletions(-) create mode 100644 grafana_dashboard.json diff --git a/README.md b/README.md index 2925fb04e..21e6da701 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,8 @@ psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' Additionally, Prometheus statistics are available at `/metrics` via HTTP. +We also have a [basic Grafana dashboard](https://github.com/postgresml/pgcat/blob/main/grafana_dashboard.json) based on Prometheus metrics that you can import into Grafana and build on it or use it for monitoring. + ### Live configuration reloading The config can be reloaded by sending a `kill -s SIGHUP` to the process or by querying `RELOAD` to the admin database. All settings except the `host` and `port` can be reloaded without restarting the pooler, including sharding and replicas configurations. diff --git a/grafana_dashboard.json b/grafana_dashboard.json new file mode 100644 index 000000000..a5f7f8357 --- /dev/null +++ b/grafana_dashboard.json @@ -0,0 +1,2124 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard to monitor PgCat deployments. It is based on the metrics exported by the Prometheus exporter that comes bundled with PgCat\n\nPlease visit https://github.com/postgresml/pgcat for more information ", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 4, + "links": [], + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 39, + "title": "Throughput Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "For this metric, each individual query outside a transaction is counted as one transaction. All queries that run inside a transaction are counted as one transaction.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(pgcat_servers_transaction_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}[1m])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Transaction Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of individual queries run", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(pgcat_servers_query_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}[1m])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Query Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Average Query Latency", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_stats_avg_query_time{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average Query Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Average latency of transactions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_stats_avg_xact_time{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average Transaction Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Data received in bytes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(pgcat_servers_bytes_received{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}[1m])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Data received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Data sent in bytes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "increase(pgcat_servers_bytes_sent{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}[1m])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Data Sent", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 22, + "panels": [], + "title": "Capacity metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "This is the ratio between the number of active server connections to the pool size. Persistently high ratio (e.g. 80%-100%) may suggest the need for a larger pool or a more performant Database", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "max": 150, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 13, + "x": 0, + "y": 14 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "(pgcat_servers_active_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"} / pgcat_databases_pool_size{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}) * 100 ", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Percentage Server Pool Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of Clients currently waiting in the queue waiting for a server connection to be assigned. This should remain close to 0. Any persistent deviation from zero means clients are blocked from querying the database", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 13, + "y": 14 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_pools_cl_waiting{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool:{{pool}}, User:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Waiting Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Banned connections won't get queries routed to them until they are unbanned. Instances are unbanned after the ban timer expires or if all replicas in the pool are banned so they are all unbanned as a failsafe. Primary instances are never banned", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 22 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_is_banned{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Banned Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "Paused connection are connections that belong to pool that was paused by the administrator", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 22 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_is_paused{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Paused Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 22 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "(pgcat_pools_maxwait{pool=~\"$pool\", user=~\"$user\"} * 1000 * 1000) + pgcat_pools_maxwait_us{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Maximum wait time by Pool", + "type": "timeseries" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 16, + "title": "Server Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 30 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_idle_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Idle Server Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 30 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_active_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Server Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 30 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_login_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Server Connection in Login State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 30 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_servers_tested_count{pool=~\"$pool\", role=~\"$role\", shard=~\"$shard_id\", index=~\"$instance_index\", host=~\"$host\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "host:{{host}}, identifier:{{role}}{{index}}, shard_id:{{shard}}, pool:{{pool}}, user:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Server Connection in Tested State", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 12, + "panels": [], + "title": "Client Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of Clients currently connected but not assigned a server connection nor are they seeking one", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 37 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_pools_cl_idle{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool:{{pool}}, User:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Idle Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of Clients currently assigned a server connection", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 37 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_pools_cl_active{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool:{{pool}}, User:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Active Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "description": "The number of Clients currently waiting in the queue waiting for a server connection to be assigned. This should remain close to 0. Any persistent deviation from zero means clients are blocked from querying the database", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 37 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "pgcat_pools_cl_waiting{pool=~\"$pool\", user=~\"$user\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool:{{pool}}, User:{{user}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Waiting Clients", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "Databases", + "PostgreSQL" + ], + "templating": { + "list": [ + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,role)", + "description": "Instance Role in the pool", + "hide": 0, + "includeAll": true, + "label": "Role", + "multi": false, + "name": "role", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,role)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,shard)", + "description": "", + "hide": 0, + "includeAll": true, + "label": "Shard ID", + "multi": true, + "name": "shard_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,shard)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,index)", + "description": "The instance index in the role (e.g. a pool with two replicas will have instances with indices 0 and 1", + "hide": 0, + "includeAll": true, + "label": "Instance Index", + "multi": true, + "name": "instance_index", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,index)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,pool)", + "description": "The PgCat Connection Pool ", + "hide": 0, + "includeAll": true, + "label": "Pool Name", + "multi": true, + "name": "pool", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,pool)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(pgcat_databases_current_connections,host)", + "description": "The underlying Database host name", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": false, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(pgcat_databases_current_connections,host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "fdwe81suzec5ce" + }, + "definition": "label_values(usename)", + "description": "PostgreSQL username used by the pool", + "hide": 0, + "includeAll": true, + "label": "Username", + "multi": true, + "name": "user", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(usename)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "PgCat Dashboard", + "uid": "ddwejyl5j6jnkb", + "version": 46, + "weekStart": "" +} diff --git a/src/prometheus.rs b/src/prometheus.rs index d8e23464e..9a14b8fec 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -12,19 +12,30 @@ use std::collections::HashMap; use std::fmt; use std::net::SocketAddr; use std::sync::atomic::Ordering; -use std::sync::Arc; use tokio::net::TcpListener; use crate::config::Address; use crate::pool::{get_all_pools, PoolIdentifier}; +use crate::stats::get_server_stats; use crate::stats::pool::PoolStats; -use crate::stats::{get_server_stats, ServerStats}; struct MetricHelpType { help: &'static str, ty: &'static str, } +struct ServerPrometheusStats { + bytes_received: u64, + bytes_sent: u64, + transaction_count: u64, + query_count: u64, + error_count: u64, + active_count: u64, + idle_count: u64, + login_count: u64, + tested_count: u64, +} + // reference for metric types: https://prometheus.io/docs/concepts/metric_types/ // counters only increase // gauges can arbitrarily increase or decrease @@ -127,22 +138,46 @@ static METRIC_HELP_AND_TYPES_LOOKUP: phf::Map<&'static str, MetricHelpType> = ph }, "servers_bytes_received" => MetricHelpType { help: "Volume in bytes of network traffic received by server", - ty: "gauge", + ty: "counter", }, "servers_bytes_sent" => MetricHelpType { help: "Volume in bytes of network traffic sent by server", - ty: "gauge", + ty: "counter", }, "servers_transaction_count" => MetricHelpType { help: "Number of transactions executed by server", - ty: "gauge", + ty: "counter", }, "servers_query_count" => MetricHelpType { help: "Number of queries executed by server", - ty: "gauge", + ty: "counter", }, "servers_error_count" => MetricHelpType { help: "Number of errors", + ty: "counter", + }, + "servers_idle_count" => MetricHelpType { + help: "Number of server connection in idle state", + ty: "gauge", + }, + "servers_active_count" => MetricHelpType { + help: "Number of server connection in active state", + ty: "gauge", + }, + "servers_tested_count" => MetricHelpType { + help: "Number of server connection in tested state", + ty: "gauge", + }, + "servers_login_count" => MetricHelpType { + help: "Number of server connection in login state", + ty: "gauge", + }, + "servers_is_banned" => MetricHelpType { + help: "0 if server is not banned, 1 if server is banned", + ty: "gauge", + }, + "servers_is_paused" => MetricHelpType { + help: "0 if server is not paused, 1 if server is paused", ty: "gauge", }, "databases_pool_size" => MetricHelpType { @@ -210,7 +245,9 @@ impl PrometheusMetric { labels.insert("shard", address.shard.to_string()); labels.insert("role", address.role.to_string()); labels.insert("pool", address.pool_name.clone()); + labels.insert("index", address.address_index.to_string()); labels.insert("database", address.database.to_string()); + labels.insert("user", address.username.clone()); Self::from_name(&format!("databases_{}", name), value, labels) } @@ -225,8 +262,9 @@ impl PrometheusMetric { labels.insert("shard", address.shard.to_string()); labels.insert("role", address.role.to_string()); labels.insert("pool", address.pool_name.clone()); + labels.insert("index", address.address_index.to_string()); labels.insert("database", address.database.to_string()); - + labels.insert("user", address.username.clone()); Self::from_name(&format!("servers_{}", name), value, labels) } @@ -236,7 +274,9 @@ impl PrometheusMetric { labels.insert("shard", address.shard.to_string()); labels.insert("pool", address.pool_name.clone()); labels.insert("role", address.role.to_string()); + labels.insert("index", address.address_index.to_string()); labels.insert("database", address.database.to_string()); + labels.insert("user", address.username.clone()); Self::from_name(&format!("stats_{}", name), value, labels) } @@ -338,34 +378,51 @@ fn push_database_stats(lines: &mut Vec) { // Adds relevant metrics shown in a SHOW SERVERS admin command. fn push_server_stats(lines: &mut Vec) { let server_stats = get_server_stats(); - let mut server_stats_by_addresses = HashMap::>::new(); + let mut prom_stats = HashMap::::new(); for (_, stats) in server_stats { - server_stats_by_addresses.insert(stats.address_name(), stats); + let entry = prom_stats + .entry(stats.address_name()) + .or_insert(ServerPrometheusStats { + bytes_received: 0, + bytes_sent: 0, + transaction_count: 0, + query_count: 0, + error_count: 0, + active_count: 0, + idle_count: 0, + login_count: 0, + tested_count: 0, + }); + entry.bytes_received += stats.bytes_received.load(Ordering::Relaxed); + entry.bytes_sent += stats.bytes_sent.load(Ordering::Relaxed); + entry.transaction_count += stats.transaction_count.load(Ordering::Relaxed); + entry.query_count += stats.query_count.load(Ordering::Relaxed); + entry.error_count += stats.error_count.load(Ordering::Relaxed); + match stats.state.load(Ordering::Relaxed) { + crate::stats::ServerState::Login => entry.login_count += 1, + crate::stats::ServerState::Active => entry.active_count += 1, + crate::stats::ServerState::Tested => entry.tested_count += 1, + crate::stats::ServerState::Idle => entry.idle_count += 1, + } } for (_, pool) in get_all_pools() { for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { let address = pool.address(shard, server); - if let Some(server_info) = server_stats_by_addresses.get(&address.name()) { + if let Some(server_info) = prom_stats.get(&address.name()) { let metrics = [ - ( - "bytes_received", - server_info.bytes_received.load(Ordering::Relaxed), - ), - ("bytes_sent", server_info.bytes_sent.load(Ordering::Relaxed)), - ( - "transaction_count", - server_info.transaction_count.load(Ordering::Relaxed), - ), - ( - "query_count", - server_info.query_count.load(Ordering::Relaxed), - ), - ( - "error_count", - server_info.error_count.load(Ordering::Relaxed), - ), + ("bytes_received", server_info.bytes_received), + ("bytes_sent", server_info.bytes_sent), + ("transaction_count", server_info.transaction_count), + ("query_count", server_info.query_count), + ("error_count", server_info.error_count), + ("idle_count", server_info.idle_count), + ("active_count", server_info.active_count), + ("login_count", server_info.login_count), + ("tested_count", server_info.tested_count), + ("is_banned", if pool.is_banned(address) { 1 } else { 0 }), + ("is_paused", if pool.paused() { 1 } else { 0 }), ]; for (key, value) in metrics { if let Some(prometheus_metric) = From 3108944c309899fe1603899a833f6db77470b833 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sat, 31 Aug 2024 10:52:33 -0500 Subject: [PATCH 510/551] Add test case for issue 776 (#786) I am adding a tiny test that uses the SQL statement that was reported to break an older version of SQL parser library #776 --- src/query_router.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/query_router.rs b/src/query_router.rs index da67e96a6..bc6ed2c66 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1399,6 +1399,19 @@ mod test { assert!(!qr.query_parser_enabled()); } + #[test] + fn test_query_parser() { + QueryRouter::setup(); + let mut qr = QueryRouter::new(); + qr.pool_settings.query_parser_read_write_splitting = true; + + let query = simple_query("SELECT req_tab_0.* FROM validation req_tab_0 WHERE array['http://www.w3.org/ns/shacl#ValidationResult'] && req_tab_0.type::text[] AND ( ( (req_tab_0.focusnode = 'DataSource_Credilogic_DataSourceAddress_144959227') ) )"); + assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); + + let query = simple_query("WITH EmployeeSalaries AS (SELECT Department, Salary FROM Employees) SELECT Department, AVG(Salary) AS AverageSalary FROM EmployeeSalaries GROUP BY Department;"); + assert!(qr.infer(&qr.parse(&query).unwrap()).is_ok()); + } + #[test] fn test_update_from_pool_settings() { QueryRouter::setup(); From fa33b2b7de97c1552e25dbb09875c2277b8e26fd Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Sat, 31 Aug 2024 17:15:13 -0500 Subject: [PATCH 511/551] Fix broken integration test #740 (#787) --- .circleci/pgcat.toml | 2 + tests/rust/Cargo.lock | 662 ++++++++++++++++++++++++++--------------- tests/rust/src/main.rs | 8 +- 3 files changed, 420 insertions(+), 252 deletions(-) diff --git a/.circleci/pgcat.toml b/.circleci/pgcat.toml index 8b87aa038..146d1f20b 100644 --- a/.circleci/pgcat.toml +++ b/.circleci/pgcat.toml @@ -59,6 +59,7 @@ admin_password = "admin_pass" # session: one server connection per connected client # transaction: one server connection per client transaction pool_mode = "transaction" +prepared_statements_cache_size = 500 # If the client doesn't specify, route traffic to # this role by default. @@ -141,6 +142,7 @@ query_parser_enabled = true query_parser_read_write_splitting = true primary_reads_enabled = true sharding_function = "pg_bigint_hash" +prepared_statements_cache_size = 500 [pools.simple_db.users.0] username = "simple_user" diff --git a/tests/rust/Cargo.lock b/tests/rust/Cargo.lock index b182ca9ef..04dcb004f 100644 --- a/tests/rust/Cargo.lock +++ b/tests/rust/Cargo.lock @@ -2,11 +2,26 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ "getrandom", "once_cell", @@ -15,20 +30,21 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "allocator-api2" -version = "0.2.15" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "atoi" @@ -41,9 +57,24 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] [[package]] name = "base64" @@ -53,9 +84,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bitflags" @@ -63,6 +94,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + [[package]] name = "block-buffer" version = "0.10.4" @@ -74,27 +111,30 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.0.79" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -104,46 +144,42 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cpufeatures" -version = "0.2.8" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] [[package]] name = "crc" -version = "3.0.1" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" dependencies = [ "crc-catalog", ] [[package]] name = "crc-catalog" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crossbeam-queue" -version = "0.3.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crypto-common" @@ -155,6 +191,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "digest" version = "0.10.7" @@ -194,9 +239,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "either" -version = "1.8.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "event-listener" @@ -206,18 +251,18 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -225,9 +270,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-intrusive" @@ -242,21 +287,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-core", "futures-sink", @@ -277,15 +322,21 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", "wasi", ] +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + [[package]] name = "hashbrown" version = "0.12.3" @@ -294,21 +345,21 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.8.3", + "ahash 0.8.11", "allocator-api2", ] [[package]] name = "hashlink" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.0", + "hashbrown 0.14.5", ] [[package]] @@ -322,12 +373,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -337,9 +385,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hkdf" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ "hmac", ] @@ -355,9 +403,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -375,9 +423,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", ] @@ -388,41 +436,42 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f84f1612606f3753f205a4e9a2efd6fe5b4c573a6269b2cc6c3003d44a0d127" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] [[package]] name = "libc" -version = "0.2.146" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", +] [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -430,24 +479,25 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "minimal-lexical" @@ -455,12 +505,22 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + [[package]] name = "mio" -version = "0.8.8" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi", "libc", "wasi", "windows-sys", @@ -476,30 +536,35 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] -name = "num_cpus" -version = "1.15.0" +name = "object" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ - "hermit-abi", - "libc", + "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "parking_lot" @@ -514,12 +579,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core 0.9.10", ] [[package]] @@ -538,34 +603,34 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.5.3", "smallvec", "windows-targets", ] [[package]] name = "paste" -version = "1.0.12" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -573,26 +638,35 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.28" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -633,26 +707,35 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ - "bitflags", + "bitflags 2.6.0", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", - "redox_syscall 0.2.16", + "libredox", "thiserror", ] @@ -665,12 +748,27 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys", +] + [[package]] name = "rust" version = "0.1.0" @@ -679,85 +777,92 @@ dependencies = [ "tokio", ] +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + [[package]] name = "rustls" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", - "ring", + "ring 0.16.20", "sct", "webpki", ] [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.2", + "base64 0.21.7", ] [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] name = "serde" -version = "1.0.164" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.77", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] [[package]] name = "sha1" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -766,38 +871,44 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "socket2" -version = "0.4.9" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "winapi", + "windows-sys", ] [[package]] @@ -806,13 +917,18 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "sqlformat" -version = "0.2.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" +checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f" dependencies = [ - "itertools", "nom", "unicode_categories", ] @@ -833,10 +949,10 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" dependencies = [ - "ahash 0.7.6", + "ahash 0.7.8", "atoi", "base64 0.13.1", - "bitflags", + "bitflags 1.3.2", "byteorder", "bytes", "crc", @@ -916,19 +1032,20 @@ dependencies = [ [[package]] name = "stringprep" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ "unicode-bidi", "unicode-normalization", + "unicode-properties", ] [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -943,9 +1060,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -954,31 +1071,34 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.77", ] [[package]] name = "time" -version = "0.3.22" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ + "deranged", "itoa", + "num-conv", + "powerfmt", "serde", "time-core", "time-macros", @@ -986,24 +1106,25 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.9" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -1016,16 +1137,15 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", "mio", - "num_cpus", - "parking_lot 0.12.1", + "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", "socket2", @@ -1035,13 +1155,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.77", ] [[package]] @@ -1057,9 +1177,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -1068,36 +1188,42 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" + [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode_categories" @@ -1111,11 +1237,17 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" -version = "2.4.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -1124,15 +1256,15 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.4" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wasi" @@ -1140,36 +1272,43 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.77", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1177,28 +1316,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -1206,12 +1345,12 @@ dependencies = [ [[package]] name = "webpki" -version = "0.22.2" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring", - "untrusted", + "ring 0.17.8", + "untrusted 0.9.0", ] [[package]] @@ -1225,11 +1364,12 @@ dependencies = [ [[package]] name = "whoami" -version = "1.4.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c70234412ca409cc04e864e89523cb0fc37f5e1344ebed5a3ebf4192b6b9f68" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "wasm-bindgen", + "redox_syscall 0.4.1", + "wasite", "web-sys", ] @@ -1257,22 +1397,23 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.48.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", "windows_i686_gnu", + "windows_i686_gnullvm", "windows_i686_msvc", "windows_x86_64_gnu", "windows_x86_64_gnullvm", @@ -1281,42 +1422,69 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] diff --git a/tests/rust/src/main.rs b/tests/rust/src/main.rs index c61d48ccd..958d75d31 100644 --- a/tests/rust/src/main.rs +++ b/tests/rust/src/main.rs @@ -15,13 +15,11 @@ async fn test_prepared_statements() { for _ in 0..5 { let pool = pool.clone(); let handle = tokio::task::spawn(async move { - for _ in 0..1000 { - match sqlx::query("SELECT one").fetch_all(&pool).await { + for i in 0..1000 { + match sqlx::query(&format!("SELECT {:?}", i % 5)).fetch_all(&pool).await { Ok(_) => (), Err(err) => { - if err.to_string().contains("prepared statement") { - panic!("prepared statement error: {}", err); - } + panic!("prepared statement error: {}", err); } } } From 847eae3951052d801f0ae2f68ab8a9960c918ab7 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Tue, 3 Sep 2024 11:15:53 -0500 Subject: [PATCH 512/551] Make iterating on integration tests easier (#789) Writing and iterating on integration tests are cumbersome, having to wait 10 minutes for the test-suite to run just to see if your test works or not is unacceptable. In this PR, I added a detailed workflow for writing tests that should shorten the feedback cycle of modifying tests to be as low as a few seconds. It will involve opening a shell into a long-lived container that has all the setup and dependencies necessary and then running your desired tests directly there. I added a convenience script that bootstraps the environment and then opens an interactive shell into the container and you can then run tests immediately in an environment that is more or less identical to what we have running in CircleCI --- CONTRIBUTING.md | 26 +++++++++++++++++++++++++ start_test_env.sh | 34 +++++++++++++++++++++++++++++++++ tests/docker/docker-compose.yml | 3 ++- tests/docker/run.sh | 32 +++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 1 deletion(-) create mode 100755 start_test_env.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0d5d160f..98c00062c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,32 @@ Thank you for contributing! Just a few tips here: 2. Run the test suite (e.g. `pgbench`) to make sure everything still works. The tests are in `.circleci/run_tests.sh`. 3. Performance is important, make sure there are no regressions in your branch vs. `main`. +## How to run the integration tests locally and iterate on them +We have integration tests written in Ruby, Python, Go and Rust. +Below are the steps to run them in a developer-friendly way that allows iterating and quick turnaround. +Hear me out, this should be easy, it will involve opening a shell into a container with all the necessary dependancies available for you and you can modify the test code and immediately rerun your test in the interactive shell. + + +Quite simply, make sure you have docker installed and then run +`./start_test_env.sh` + +That is it! + +Within this test environment you can modify the file in your favorite IDE and rerun the tests without having to bootstrap the entire environment again. + +Once the environment is ready, you can run the tests by running +Ruby: `cd /app/tests/ruby && bundle exec ruby .rb --format documentation` +Python: `cd /app && python3 tests/python/tests.py` +Rust: `cd /app/tests/rust && cargo run` +Go: `cd /app/tests/go && /usr/local/go/bin/go test` + +You can also rebuild PgCat directly within the environment and the tests will run against the newly built binary +To rebuild PgCat, just run `cargo build` within the container under `/app` + +![Animated gif showing how to run tests](https://github.com/user-attachments/assets/2258fde3-2aed-4efb-bdc5-e4f12dcd4d33) + + + Happy hacking! ## TODOs diff --git a/start_test_env.sh b/start_test_env.sh new file mode 100755 index 000000000..24a6b3fb4 --- /dev/null +++ b/start_test_env.sh @@ -0,0 +1,34 @@ +GREEN="\033[0;32m" +RED="\033[0;31m" +BLUE="\033[0;34m" +RESET="\033[0m" + + +cd tests/docker/ +docker compose kill main || true +docker compose build main +docker compose down +docker compose up -d +# wait for the container to start +while ! docker compose exec main ls; do + echo "Waiting for test environment to start" + sleep 1 +done +echo "===================================" +docker compose exec -e LOG_LEVEL=error -d main toxiproxy-server +docker compose exec --workdir /app main cargo build +docker compose exec -d --workdir /app main ./target/debug/pgcat ./.circleci/pgcat.toml +docker compose exec --workdir /app/tests/ruby main bundle install +docker compose exec --workdir /app/tests/python main pip3 install -r requirements.txt +echo "Interactive test environment ready" +echo "To run integration tests, you can use the following commands:" +echo -e " ${BLUE}Ruby: ${RED}cd /app/tests/ruby && bundle exec ruby tests.rb --format documentation${RESET}" +echo -e " ${BLUE}Python: ${RED}cd /app && python3 tests/python/tests.py${RESET}" +echo -e " ${BLUE}Rust: ${RED}cd /app/tests/rust && cargo run ${RESET}" +echo -e " ${BLUE}Go: ${RED}cd /app/tests/go && /usr/local/go/bin/go test${RESET}" +echo "the source code for tests are directly linked to the source code in the container so you can modify the code and run the tests again" +echo "You can rebuild PgCat from within the container by running" +echo -e " ${GREEN}cargo build${RESET}" +echo "and then run the tests again" +echo "===================================" +docker compose exec --workdir /app/tests main bash diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index 93e945502..0e174d839 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3" services: pg1: image: postgres:14 @@ -48,6 +47,8 @@ services: main: build: . command: ["bash", "/app/tests/docker/run.sh"] + environment: + - INTERACTIVE_TEST_ENVIRONMENT=true volumes: - ../../:/app/ - /app/target/ diff --git a/tests/docker/run.sh b/tests/docker/run.sh index ae30c9784..07452cbe4 100644 --- a/tests/docker/run.sh +++ b/tests/docker/run.sh @@ -5,6 +5,38 @@ rm /app/*.profraw || true rm /app/pgcat.profdata || true rm -rf /app/cov || true +# Prepares the interactive test environment +# +if [ -n "$INTERACTIVE_TEST_ENVIRONMENT" ]; then + ports=(5432 7432 8432 9432 10432) + for port in "${ports[@]}"; do + is_it_up=0 + attempts=0 + while [ $is_it_up -eq 0 ]; do + PGPASSWORD=postgres psql -h 127.0.0.1 -p $port -U postgres -c '\q' > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "PostgreSQL on port $port is up." + is_it_up=1 + else + attempts=$((attempts+1)) + if [ $attempts -gt 10 ]; then + echo "PostgreSQL on port $port is down, giving up." + exit 1 + fi + echo "PostgreSQL on port $port is down, waiting for it to start." + sleep 1 + fi + done + done + PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 5432 -U postgres -f /app/tests/sharding/query_routing_setup.sql + PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 7432 -U postgres -f /app/tests/sharding/query_routing_setup.sql + PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 8432 -U postgres -f /app/tests/sharding/query_routing_setup.sql + PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 9432 -U postgres -f /app/tests/sharding/query_routing_setup.sql + PGPASSWORD=postgres psql -e -h 127.0.0.1 -p 10432 -U postgres -f /app/tests/sharding/query_routing_setup.sql + sleep 100000000000000000 + exit 0 +fi + export LLVM_PROFILE_FILE="/app/pgcat-%m-%p.profraw" export RUSTC_BOOTSTRAP=1 export CARGO_INCREMENTAL=0 From 31418d58f2fc22fcf35ca03b131802f22615f66a Mon Sep 17 00:00:00 2001 From: Andrew Jackson <46945903+AndrewJackson2020@users.noreply.github.com> Date: Thu, 5 Sep 2024 08:01:33 -0500 Subject: [PATCH 513/551] Fix CI script to allow consecutive runs locally (#793) --- .circleci/run_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index b6bc42281..41aae912e 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -26,6 +26,7 @@ PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard1 -i PGPASSWORD=sharding_user pgbench -h 127.0.0.1 -U sharding_user shard2 -i # Start Toxiproxy +kill -9 $(pgrep toxiproxy) || true LOG_LEVEL=error toxiproxy-server & sleep 1 @@ -177,3 +178,6 @@ killall pgcat -s SIGINT # Allow for graceful shutdown sleep 1 + +kill -9 $(pgrep toxiproxy) +sleep 1 From 1d58a42d07f32919eaff97c923f3ff472ee78285 Mon Sep 17 00:00:00 2001 From: Andrew Jackson <46945903+AndrewJackson2020@users.noreply.github.com> Date: Thu, 5 Sep 2024 08:16:45 -0500 Subject: [PATCH 514/551] Implemented python tests with pytest (#790) Currently the python tests act as scripts. A lot of output is generated to stdout which makes it very hard to figure out where problems were. Also if you want to run only a single test you basically need to comment out code in order to accomplish this. This PR modifies the python tests to us the pytest python testing framework. This framework allows individual tests to be targeted via the command line, without touching the source code. It also suppressed stdout by default making the test output much easier to read. Also after the tests run it will provide a summary of what failed, what succeded, etc. --- .circleci/run_tests.sh | 2 +- .gitignore | 3 +- CONTRIBUTING.md | 4 +- start_test_env.sh | 4 +- tests/python/conftest.py | 0 tests/python/requirements.txt | 3 +- tests/python/{tests.py => test_pgcat.py} | 173 ++++++++--------------- tests/python/utils.py | 60 ++++++++ 8 files changed, 125 insertions(+), 124 deletions(-) create mode 100644 tests/python/conftest.py rename tests/python/{tests.py => test_pgcat.py} (61%) create mode 100644 tests/python/utils.py diff --git a/.circleci/run_tests.sh b/.circleci/run_tests.sh index 41aae912e..a93b484d3 100644 --- a/.circleci/run_tests.sh +++ b/.circleci/run_tests.sh @@ -107,7 +107,7 @@ cd ../.. # These tests will start and stop the pgcat server so it will need to be restarted after the tests # pip3 install -r tests/python/requirements.txt -python3 tests/python/tests.py || exit 1 +pytest || exit 1 # diff --git a/.gitignore b/.gitignore index 40d5de1b9..ae1348147 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ lcov.info dev/.bash_history dev/cache !dev/cache/.keepme -.venv \ No newline at end of file +.venv +**/__pycache__ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98c00062c..b94680135 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Thank you for contributing! Just a few tips here: 3. Performance is important, make sure there are no regressions in your branch vs. `main`. ## How to run the integration tests locally and iterate on them -We have integration tests written in Ruby, Python, Go and Rust. +We have integration tests written in Ruby, Python, Go and Rust. Below are the steps to run them in a developer-friendly way that allows iterating and quick turnaround. Hear me out, this should be easy, it will involve opening a shell into a container with all the necessary dependancies available for you and you can modify the test code and immediately rerun your test in the interactive shell. @@ -21,7 +21,7 @@ Within this test environment you can modify the file in your favorite IDE and re Once the environment is ready, you can run the tests by running Ruby: `cd /app/tests/ruby && bundle exec ruby .rb --format documentation` -Python: `cd /app && python3 tests/python/tests.py` +Python: `cd /app/ && pytest` Rust: `cd /app/tests/rust && cargo run` Go: `cd /app/tests/go && /usr/local/go/bin/go test` diff --git a/start_test_env.sh b/start_test_env.sh index 24a6b3fb4..c7ec4b13a 100755 --- a/start_test_env.sh +++ b/start_test_env.sh @@ -23,11 +23,11 @@ docker compose exec --workdir /app/tests/python main pip3 install -r requirement echo "Interactive test environment ready" echo "To run integration tests, you can use the following commands:" echo -e " ${BLUE}Ruby: ${RED}cd /app/tests/ruby && bundle exec ruby tests.rb --format documentation${RESET}" -echo -e " ${BLUE}Python: ${RED}cd /app && python3 tests/python/tests.py${RESET}" +echo -e " ${BLUE}Python: ${RED}cd /app/ && pytest ${RESET}" echo -e " ${BLUE}Rust: ${RED}cd /app/tests/rust && cargo run ${RESET}" echo -e " ${BLUE}Go: ${RED}cd /app/tests/go && /usr/local/go/bin/go test${RESET}" echo "the source code for tests are directly linked to the source code in the container so you can modify the code and run the tests again" -echo "You can rebuild PgCat from within the container by running" +echo "You can rebuild PgCat from within the container by running" echo -e " ${GREEN}cargo build${RESET}" echo "and then run the tests again" echo "===================================" diff --git a/tests/python/conftest.py b/tests/python/conftest.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/python/requirements.txt b/tests/python/requirements.txt index eebd9c90a..ec8b08f81 100644 --- a/tests/python/requirements.txt +++ b/tests/python/requirements.txt @@ -1,2 +1,3 @@ +pytest psycopg2==2.9.3 -psutil==5.9.1 \ No newline at end of file +psutil==5.9.1 diff --git a/tests/python/tests.py b/tests/python/test_pgcat.py similarity index 61% rename from tests/python/tests.py rename to tests/python/test_pgcat.py index cd54081f4..dc2f11e5f 100644 --- a/tests/python/tests.py +++ b/tests/python/test_pgcat.py @@ -1,83 +1,29 @@ -from typing import Tuple -import psycopg2 -import psutil import os import signal import time -SHUTDOWN_TIMEOUT = 5 - -PGCAT_HOST = "127.0.0.1" -PGCAT_PORT = "6432" - - -def pgcat_start(): - pg_cat_send_signal(signal.SIGTERM) - os.system("./target/debug/pgcat .circleci/pgcat.toml &") - time.sleep(2) - - -def pg_cat_send_signal(signal: signal.Signals): - try: - for proc in psutil.process_iter(["pid", "name"]): - if "pgcat" == proc.name(): - os.kill(proc.pid, signal) - except Exception as e: - # The process can be gone when we send this signal - print(e) - - if signal == signal.SIGTERM: - # Returns 0 if pgcat process exists - time.sleep(2) - if not os.system('pgrep pgcat'): - raise Exception("pgcat not closed after SIGTERM") - - -def connect_db( - autocommit: bool = True, - admin: bool = False, -) -> Tuple[psycopg2.extensions.connection, psycopg2.extensions.cursor]: - - if admin: - user = "admin_user" - password = "admin_pass" - db = "pgcat" - else: - user = "sharding_user" - password = "sharding_user" - db = "sharded_db" - - conn = psycopg2.connect( - f"postgres://{user}:{password}@{PGCAT_HOST}:{PGCAT_PORT}/{db}?application_name=testing_pgcat", - connect_timeout=2, - ) - conn.autocommit = autocommit - cur = conn.cursor() - - return (conn, cur) - +import psycopg2 -def cleanup_conn(conn: psycopg2.extensions.connection, cur: psycopg2.extensions.cursor): - cur.close() - conn.close() +import utils +SHUTDOWN_TIMEOUT = 5 def test_normal_db_access(): - pgcat_start() - conn, cur = connect_db(autocommit=False) + utils.pgcat_start() + conn, cur = utils.connect_db(autocommit=False) cur.execute("SELECT 1") res = cur.fetchall() print(res) - cleanup_conn(conn, cur) + utils.cleanup_conn(conn, cur) def test_admin_db_access(): - conn, cur = connect_db(admin=True) + conn, cur = utils.connect_db(admin=True) cur.execute("SHOW POOLS") res = cur.fetchall() print(res) - cleanup_conn(conn, cur) + utils.cleanup_conn(conn, cur) def test_shutdown_logic(): @@ -86,17 +32,17 @@ def test_shutdown_logic(): # NO ACTIVE QUERIES SIGINT HANDLING # Start pgcat - pgcat_start() + utils.pgcat_start() # Create client connection and send query (not in transaction) - conn, cur = connect_db() + conn, cur = utils.connect_db() cur.execute("BEGIN;") cur.execute("SELECT 1;") cur.execute("COMMIT;") # Send sigint to pgcat - pg_cat_send_signal(signal.SIGINT) + utils.pg_cat_send_signal(signal.SIGINT) time.sleep(1) # Check that any new queries fail after sigint since server should close with no active transactions @@ -108,18 +54,18 @@ def test_shutdown_logic(): # Fail if query execution succeeded raise Exception("Server not closed after sigint") - cleanup_conn(conn, cur) - pg_cat_send_signal(signal.SIGTERM) + utils.cleanup_conn(conn, cur) + utils.pg_cat_send_signal(signal.SIGTERM) # - - - - - - - - - - - - - - - - - - # NO ACTIVE QUERIES ADMIN SHUTDOWN COMMAND # Start pgcat - pgcat_start() + utils.pgcat_start() # Create client connection and begin transaction - conn, cur = connect_db() - admin_conn, admin_cur = connect_db(admin=True) + conn, cur = utils.connect_db() + admin_conn, admin_cur = utils.connect_db(admin=True) cur.execute("BEGIN;") cur.execute("SELECT 1;") @@ -138,24 +84,24 @@ def test_shutdown_logic(): # Fail if query execution succeeded raise Exception("Server not closed after sigint") - cleanup_conn(conn, cur) - cleanup_conn(admin_conn, admin_cur) - pg_cat_send_signal(signal.SIGTERM) + utils.cleanup_conn(conn, cur) + utils.cleanup_conn(admin_conn, admin_cur) + utils.pg_cat_send_signal(signal.SIGTERM) # - - - - - - - - - - - - - - - - - - # HANDLE TRANSACTION WITH SIGINT # Start pgcat - pgcat_start() + utils.pgcat_start() # Create client connection and begin transaction - conn, cur = connect_db() + conn, cur = utils.connect_db() cur.execute("BEGIN;") cur.execute("SELECT 1;") # Send sigint to pgcat while still in transaction - pg_cat_send_signal(signal.SIGINT) + utils.pg_cat_send_signal(signal.SIGINT) time.sleep(1) # Check that any new queries succeed after sigint since server should still allow transaction to complete @@ -165,18 +111,18 @@ def test_shutdown_logic(): # Fail if query fails since server closed raise Exception("Server closed while in transaction", e.pgerror) - cleanup_conn(conn, cur) - pg_cat_send_signal(signal.SIGTERM) + utils.cleanup_conn(conn, cur) + utils.pg_cat_send_signal(signal.SIGTERM) # - - - - - - - - - - - - - - - - - - # HANDLE TRANSACTION WITH ADMIN SHUTDOWN COMMAND # Start pgcat - pgcat_start() + utils.pgcat_start() # Create client connection and begin transaction - conn, cur = connect_db() - admin_conn, admin_cur = connect_db(admin=True) + conn, cur = utils.connect_db() + admin_conn, admin_cur = utils.connect_db(admin=True) cur.execute("BEGIN;") cur.execute("SELECT 1;") @@ -194,30 +140,30 @@ def test_shutdown_logic(): # Fail if query fails since server closed raise Exception("Server closed while in transaction", e.pgerror) - cleanup_conn(conn, cur) - cleanup_conn(admin_conn, admin_cur) - pg_cat_send_signal(signal.SIGTERM) + utils.cleanup_conn(conn, cur) + utils.cleanup_conn(admin_conn, admin_cur) + utils.pg_cat_send_signal(signal.SIGTERM) # - - - - - - - - - - - - - - - - - - # NO NEW NON-ADMIN CONNECTIONS DURING SHUTDOWN # Start pgcat - pgcat_start() + utils.pgcat_start() # Create client connection and begin transaction - transaction_conn, transaction_cur = connect_db() + transaction_conn, transaction_cur = utils.connect_db() transaction_cur.execute("BEGIN;") transaction_cur.execute("SELECT 1;") # Send sigint to pgcat while still in transaction - pg_cat_send_signal(signal.SIGINT) + utils.pg_cat_send_signal(signal.SIGINT) time.sleep(1) start = time.perf_counter() try: - conn, cur = connect_db() + conn, cur = utils.connect_db() cur.execute("SELECT 1;") - cleanup_conn(conn, cur) + utils.cleanup_conn(conn, cur) except psycopg2.OperationalError as e: time_taken = time.perf_counter() - start if time_taken > 0.1: @@ -227,49 +173,49 @@ def test_shutdown_logic(): else: raise Exception("Able connect to database during shutdown") - cleanup_conn(transaction_conn, transaction_cur) - pg_cat_send_signal(signal.SIGTERM) + utils.cleanup_conn(transaction_conn, transaction_cur) + utils.pg_cat_send_signal(signal.SIGTERM) # - - - - - - - - - - - - - - - - - - # ALLOW NEW ADMIN CONNECTIONS DURING SHUTDOWN # Start pgcat - pgcat_start() + utils.pgcat_start() # Create client connection and begin transaction - transaction_conn, transaction_cur = connect_db() + transaction_conn, transaction_cur = utils.connect_db() transaction_cur.execute("BEGIN;") transaction_cur.execute("SELECT 1;") # Send sigint to pgcat while still in transaction - pg_cat_send_signal(signal.SIGINT) + utils.pg_cat_send_signal(signal.SIGINT) time.sleep(1) try: - conn, cur = connect_db(admin=True) + conn, cur = utils.connect_db(admin=True) cur.execute("SHOW DATABASES;") - cleanup_conn(conn, cur) + utils.cleanup_conn(conn, cur) except psycopg2.OperationalError as e: raise Exception(e) - cleanup_conn(transaction_conn, transaction_cur) - pg_cat_send_signal(signal.SIGTERM) + utils.cleanup_conn(transaction_conn, transaction_cur) + utils.pg_cat_send_signal(signal.SIGTERM) # - - - - - - - - - - - - - - - - - - # ADMIN CONNECTIONS CONTINUING TO WORK AFTER SHUTDOWN # Start pgcat - pgcat_start() + utils.pgcat_start() # Create client connection and begin transaction - transaction_conn, transaction_cur = connect_db() + transaction_conn, transaction_cur = utils.connect_db() transaction_cur.execute("BEGIN;") transaction_cur.execute("SELECT 1;") - admin_conn, admin_cur = connect_db(admin=True) + admin_conn, admin_cur = utils.connect_db(admin=True) admin_cur.execute("SHOW DATABASES;") # Send sigint to pgcat while still in transaction - pg_cat_send_signal(signal.SIGINT) + utils.pg_cat_send_signal(signal.SIGINT) time.sleep(1) try: @@ -277,24 +223,24 @@ def test_shutdown_logic(): except psycopg2.OperationalError as e: raise Exception("Could not execute admin command:", e) - cleanup_conn(transaction_conn, transaction_cur) - cleanup_conn(admin_conn, admin_cur) - pg_cat_send_signal(signal.SIGTERM) + utils.cleanup_conn(transaction_conn, transaction_cur) + utils.cleanup_conn(admin_conn, admin_cur) + utils.pg_cat_send_signal(signal.SIGTERM) # - - - - - - - - - - - - - - - - - - # HANDLE SHUTDOWN TIMEOUT WITH SIGINT # Start pgcat - pgcat_start() + utils.pgcat_start() # Create client connection and begin transaction, which should prevent server shutdown unless shutdown timeout is reached - conn, cur = connect_db() + conn, cur = utils.connect_db() cur.execute("BEGIN;") cur.execute("SELECT 1;") # Send sigint to pgcat while still in transaction - pg_cat_send_signal(signal.SIGINT) + utils.pg_cat_send_signal(signal.SIGINT) # pgcat shutdown timeout is set to SHUTDOWN_TIMEOUT seconds, so we sleep for SHUTDOWN_TIMEOUT + 1 seconds time.sleep(SHUTDOWN_TIMEOUT + 1) @@ -308,12 +254,5 @@ def test_shutdown_logic(): # Fail if query execution succeeded raise Exception("Server not closed after sigint and expected timeout") - cleanup_conn(conn, cur) - pg_cat_send_signal(signal.SIGTERM) - - # - - - - - - - - - - - - - - - - - - - - -test_normal_db_access() -test_admin_db_access() -test_shutdown_logic() + utils.cleanup_conn(conn, cur) + utils.pg_cat_send_signal(signal.SIGTERM) diff --git a/tests/python/utils.py b/tests/python/utils.py new file mode 100644 index 000000000..5c49bce9c --- /dev/null +++ b/tests/python/utils.py @@ -0,0 +1,60 @@ +from typing import Tuple +import os +import psutil +import signal +import time + +import psycopg2 + +PGCAT_HOST = "127.0.0.1" +PGCAT_PORT = "6432" + +def pgcat_start(): + pg_cat_send_signal(signal.SIGTERM) + os.system("./target/debug/pgcat .circleci/pgcat.toml &") + time.sleep(2) + + +def pg_cat_send_signal(signal: signal.Signals): + try: + for proc in psutil.process_iter(["pid", "name"]): + if "pgcat" == proc.name(): + os.kill(proc.pid, signal) + except Exception as e: + # The process can be gone when we send this signal + print(e) + + if signal == signal.SIGTERM: + # Returns 0 if pgcat process exists + time.sleep(2) + if not os.system('pgrep pgcat'): + raise Exception("pgcat not closed after SIGTERM") + + +def connect_db( + autocommit: bool = True, + admin: bool = False, +) -> Tuple[psycopg2.extensions.connection, psycopg2.extensions.cursor]: + + if admin: + user = "admin_user" + password = "admin_pass" + db = "pgcat" + else: + user = "sharding_user" + password = "sharding_user" + db = "sharded_db" + + conn = psycopg2.connect( + f"postgres://{user}:{password}@{PGCAT_HOST}:{PGCAT_PORT}/{db}?application_name=testing_pgcat", + connect_timeout=2, + ) + conn.autocommit = autocommit + cur = conn.cursor() + + return (conn, cur) + + +def cleanup_conn(conn: psycopg2.extensions.connection, cur: psycopg2.extensions.cursor): + cur.close() + conn.close() From 358c445f6699bd2b135059bc1ac4f10bf500f0a6 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Thu, 5 Sep 2024 08:58:18 -0500 Subject: [PATCH 515/551] Improve Prometheus exporter output (#795) * Prometheus metrics updates: * Add username label to deconflict metrics that would otherwise have duplicate labels across different pools. * Group metrics by name and only print HELP and TYPE once per metric name. * Sort labels for a deterministic output. --------- --- src/prometheus.rs | 83 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/src/prometheus.rs b/src/prometheus.rs index 9a14b8fec..93348b853 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -200,18 +200,17 @@ struct PrometheusMetric { impl fmt::Display for PrometheusMetric { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let formatted_labels = self - .labels + let mut sorted_labels: Vec<_> = self.labels.iter().collect(); + sorted_labels.sort_by_key(|&(key, _)| key); + let formatted_labels = sorted_labels .iter() .map(|(key, value)| format!("{}=\"{}\"", key, value)) .collect::>() .join(","); write!( f, - "# HELP {name} {help}\n# TYPE {name} {ty}\n{name}{{{formatted_labels}}} {value}\n", + "{name}{{{formatted_labels}}} {value}", name = format_args!("pgcat_{}", self.name), - help = self.help, - ty = self.ty, formatted_labels = formatted_labels, value = self.value ) @@ -247,7 +246,7 @@ impl PrometheusMetric { labels.insert("pool", address.pool_name.clone()); labels.insert("index", address.address_index.to_string()); labels.insert("database", address.database.to_string()); - labels.insert("user", address.username.clone()); + labels.insert("username", address.username.clone()); Self::from_name(&format!("databases_{}", name), value, labels) } @@ -264,7 +263,8 @@ impl PrometheusMetric { labels.insert("pool", address.pool_name.clone()); labels.insert("index", address.address_index.to_string()); labels.insert("database", address.database.to_string()); - labels.insert("user", address.username.clone()); + labels.insert("username", address.username.clone()); + Self::from_name(&format!("servers_{}", name), value, labels) } @@ -276,7 +276,7 @@ impl PrometheusMetric { labels.insert("role", address.role.to_string()); labels.insert("index", address.address_index.to_string()); labels.insert("database", address.database.to_string()); - labels.insert("user", address.username.clone()); + labels.insert("username", address.username.clone()); Self::from_name(&format!("stats_{}", name), value, labels) } @@ -288,6 +288,15 @@ impl PrometheusMetric { Self::from_name(&format!("pools_{}", name), value, labels) } + + fn get_header(&self) -> String { + format!( + "\n# HELP {name} {help}\n# TYPE {name} {ty}", + name = format_args!("pgcat_{}", self.name), + help = self.help, + ty = self.ty, + ) + } } async fn prometheus_stats( @@ -313,6 +322,7 @@ async fn prometheus_stats( // Adds metrics shown in a SHOW STATS admin command. fn push_address_stats(lines: &mut Vec) { + let mut grouped_metrics: HashMap>> = HashMap::new(); for (_, pool) in get_all_pools() { for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { @@ -322,7 +332,10 @@ fn push_address_stats(lines: &mut Vec) { if let Some(prometheus_metric) = PrometheusMetric::::from_address(address, &key, value) { - lines.push(prometheus_metric.to_string()); + grouped_metrics + .entry(key) + .or_default() + .push(prometheus_metric); } else { debug!("Metric {} not implemented for {}", key, address.name()); } @@ -330,33 +343,53 @@ fn push_address_stats(lines: &mut Vec) { } } } + for (_key, metrics) in grouped_metrics { + if !metrics.is_empty() { + lines.push(metrics[0].get_header()); + for metric in metrics { + lines.push(metric.to_string()); + } + } + } } // Adds relevant metrics shown in a SHOW POOLS admin command. fn push_pool_stats(lines: &mut Vec) { + let mut grouped_metrics: HashMap>> = HashMap::new(); let pool_stats = PoolStats::construct_pool_lookup(); for (pool_id, stats) in pool_stats.iter() { for (name, value) in stats.clone() { if let Some(prometheus_metric) = PrometheusMetric::::from_pool(pool_id.clone(), &name, value) { - lines.push(prometheus_metric.to_string()); + grouped_metrics + .entry(name) + .or_default() + .push(prometheus_metric); } else { debug!("Metric {} not implemented for ({})", name, *pool_id); } } } + for (_key, metrics) in grouped_metrics { + if !metrics.is_empty() { + lines.push(metrics[0].get_header()); + for metric in metrics { + lines.push(metric.to_string()); + } + } + } } // Adds relevant metrics shown in a SHOW DATABASES admin command. fn push_database_stats(lines: &mut Vec) { + let mut grouped_metrics: HashMap>> = HashMap::new(); for (_, pool) in get_all_pools() { let pool_config = pool.settings.clone(); for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { let address = pool.address(shard, server); let pool_state = pool.pool_state(shard, server); - let metrics = vec![ ("pool_size", pool_config.user.pool_size), ("current_connections", pool_state.connections), @@ -365,7 +398,10 @@ fn push_database_stats(lines: &mut Vec) { if let Some(prometheus_metric) = PrometheusMetric::::from_database_info(address, key, value) { - lines.push(prometheus_metric.to_string()); + grouped_metrics + .entry(key.to_string()) + .or_default() + .push(prometheus_metric); } else { debug!("Metric {} not implemented for {}", key, address.name()); } @@ -373,6 +409,14 @@ fn push_database_stats(lines: &mut Vec) { } } } + for (_key, metrics) in grouped_metrics { + if !metrics.is_empty() { + lines.push(metrics[0].get_header()); + for metric in metrics { + lines.push(metric.to_string()); + } + } + } } // Adds relevant metrics shown in a SHOW SERVERS admin command. @@ -405,7 +449,7 @@ fn push_server_stats(lines: &mut Vec) { crate::stats::ServerState::Idle => entry.idle_count += 1, } } - + let mut grouped_metrics: HashMap>> = HashMap::new(); for (_, pool) in get_all_pools() { for shard in 0..pool.shards() { for server in 0..pool.servers(shard) { @@ -428,7 +472,10 @@ fn push_server_stats(lines: &mut Vec) { if let Some(prometheus_metric) = PrometheusMetric::::from_server_info(address, key, value) { - lines.push(prometheus_metric.to_string()); + grouped_metrics + .entry(key.to_string()) + .or_default() + .push(prometheus_metric); } else { debug!("Metric {} not implemented for {}", key, address.name()); } @@ -437,6 +484,14 @@ fn push_server_stats(lines: &mut Vec) { } } } + for (_key, metrics) in grouped_metrics { + if !metrics.is_empty() { + lines.push(metrics[0].get_header()); + for metric in metrics { + lines.push(metric.to_string()); + } + } + } } pub async fn start_metric_server(http_addr: SocketAddr) { From e8fb7c653de1cea5a2e88d2a0b135821ddfaeb7a Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Fri, 6 Sep 2024 16:07:01 +0200 Subject: [PATCH 516/551] Automatically start systemd service if config file is present (#800) Previously, upgrading the deb package stopped the service but didn't reenable it after a successful upgrade. This made upgrading the package more difficult and required a second step to restart the service. With this commit, the systemd service is automatically started when the default config file is present. --- postinst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/postinst b/postinst index 50138c2b2..520599ac1 100644 --- a/postinst +++ b/postinst @@ -7,3 +7,7 @@ systemctl enable pgcat if ! id pgcat 2> /dev/null; then useradd -s /usr/bin/false pgcat fi + +if [ -f /etc/pgcat.toml ]; then + systemctl start pgcat +fi From 54736b59312385b9c23a6a57769e77e66a17f006 Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Fri, 6 Sep 2024 16:11:52 +0200 Subject: [PATCH 517/551] Automatically build deb package on a new version tag (#801) In #796, I noticed that the deb package was not build since an automation was missing. With this PR, I add the missing automation. I tested the workflow in my repo... when starting the workflow manually: https://github.com/MrSerth/pgcat/actions/runs/10737879151/job/29780286094 when drafting a new release: https://github.com/MrSerth/pgcat/actions/runs/10737835796/job/29780146212 Obviously, both workflows failed since I cannot upload to the APT repo. However, the version substitution for the workflow is working correctly (as shown when collapsing the first line of the "Build and release package" step). --- .github/workflows/publish-deb-package.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-deb-package.yml b/.github/workflows/publish-deb-package.yml index 772945287..dbb7abd84 100644 --- a/.github/workflows/publish-deb-package.yml +++ b/.github/workflows/publish-deb-package.yml @@ -1,6 +1,9 @@ name: pgcat package (deb) on: + push: + tags: + - v* workflow_dispatch: inputs: packageVersion: @@ -16,6 +19,14 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 + - name: Set package version + if: github.event_name == 'push' # For push event + run: | + TAG=${{ github.ref_name }} + echo "packageVersion=${TAG#v}" >> "$GITHUB_ENV" + - name: Set package version (manual dispatch) + if: github.event_name == 'workflow_dispatch' # For manual dispatch + run: echo "packageVersion=${{ github.event.inputs.packageVersion }}" >> "$GITHUB_ENV" - uses: actions-rs/toolchain@v1 with: toolchain: stable @@ -39,10 +50,10 @@ jobs: export ARCH=arm64 fi - bash utilities/deb.sh ${{ inputs.packageVersion }} + bash utilities/deb.sh ${{ env.packageVersion }} deb-s3 upload \ --lock \ --bucket apt.postgresml.org \ - pgcat-${{ inputs.packageVersion }}-ubuntu22.04-${ARCH}.deb \ + pgcat-${{ env.packageVersion }}-ubuntu22.04-${ARCH}.deb \ --codename $(lsb_release -cs) From 37f094bc75e320c5fdf68011c1ca8b5df24def22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Sep 2024 12:20:01 -0500 Subject: [PATCH 518/551] chore(deps): bump rexml from 3.2.5 to 3.2.8 in /tests/ruby (#743) Bumps [rexml](https://github.com/ruby/rexml) from 3.2.5 to 3.2.8. - [Release notes](https://github.com/ruby/rexml/releases) - [Changelog](https://github.com/ruby/rexml/blob/master/NEWS.md) - [Commits](https://github.com/ruby/rexml/compare/v3.2.5...v3.2.8) --- updated-dependencies: - dependency-name: rexml dependency-type: indirect ... Signed-off-by: dependabot[bot] --- tests/ruby/Gemfile.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock index f49468053..048683b14 100644 --- a/tests/ruby/Gemfile.lock +++ b/tests/ruby/Gemfile.lock @@ -24,7 +24,8 @@ GEM pg (1.3.2) rainbow (3.1.1) regexp_parser (2.3.1) - rexml (3.2.5) + rexml (3.2.8) + strscan (>= 3.0.9) rspec (3.11.0) rspec-core (~> 3.11.0) rspec-expectations (~> 3.11.0) @@ -50,6 +51,7 @@ GEM rubocop-ast (1.17.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) + strscan (3.1.0) toml (0.3.0) parslet (>= 1.8.0, < 3.0.0) toxiproxy (2.0.1) From 323d9e6754a548b4b66cfee5fdf027e326b23ee4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Sep 2024 12:20:38 -0500 Subject: [PATCH 519/551] chore(deps): bump helm/kind-action from 1.7.0 to 1.10.0 (#732) Bumps [helm/kind-action](https://github.com/helm/kind-action) from 1.7.0 to 1.10.0. - [Release notes](https://github.com/helm/kind-action/releases) - [Commits](https://github.com/helm/kind-action/compare/v1.7.0...v1.10.0) --- updated-dependencies: - dependency-name: helm/kind-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/chart-lint-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/chart-lint-test.yaml b/.github/workflows/chart-lint-test.yaml index c5a8508cd..50cdfa718 100644 --- a/.github/workflows/chart-lint-test.yaml +++ b/.github/workflows/chart-lint-test.yaml @@ -43,7 +43,7 @@ jobs: run: ct lint --config ct.yaml - name: Create kind cluster - uses: helm/kind-action@v1.7.0 + uses: helm/kind-action@v1.10.0 if: steps.list-changed.outputs.changed == 'true' - name: Run chart-testing (install) From a94e879ec94dd5b4298ea51c5000715d29bd8c92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Sep 2024 12:21:21 -0500 Subject: [PATCH 520/551] chore(deps): bump actions/setup-python from 4.1.0 to 5.1.0 (#715) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.1.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.1.0...v5.1.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/chart-lint-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/chart-lint-test.yaml b/.github/workflows/chart-lint-test.yaml index 50cdfa718..c19789fba 100644 --- a/.github/workflows/chart-lint-test.yaml +++ b/.github/workflows/chart-lint-test.yaml @@ -22,7 +22,7 @@ jobs: # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and # yamllint (https://github.com/adrienverge/yamllint) which require Python - name: Set up Python - uses: actions/setup-python@v4.1.0 + uses: actions/setup-python@v5.1.0 with: python-version: 3.7 From 872f742c153ca3b051b9460463b74b9bb35609d4 Mon Sep 17 00:00:00 2001 From: Andrew Jackson <46945903+AndrewJackson2020@users.noreply.github.com> Date: Tue, 10 Sep 2024 09:29:45 -0500 Subject: [PATCH 521/551] Implement Trust Authentication (#805) * Implement Trust Authentication * Remove remaining LDAP stuff * Reverted LDAP changes, Cleaned up tests --------- --- src/auth_passthrough.rs | 2 + src/client.rs | 311 ++++++++++++++++++++++--------------- src/config.rs | 25 +++ tests/python/conftest.py | 0 tests/python/test_auth.py | 71 +++++++++ tests/python/test_pgcat.py | 22 +-- tests/python/utils.py | 58 ++++++- 7 files changed, 339 insertions(+), 150 deletions(-) delete mode 100644 tests/python/conftest.py create mode 100644 tests/python/test_auth.py diff --git a/src/auth_passthrough.rs b/src/auth_passthrough.rs index 159847ed2..53ef93d48 100644 --- a/src/auth_passthrough.rs +++ b/src/auth_passthrough.rs @@ -1,3 +1,4 @@ +use crate::config::AuthType; use crate::errors::Error; use crate::pool::ConnectionPool; use crate::server::Server; @@ -71,6 +72,7 @@ impl AuthPassthrough { pub async fn fetch_hash(&self, address: &crate::config::Address) -> Result { let auth_user = crate::config::User { username: self.user.clone(), + auth_type: AuthType::MD5, password: Some(self.password.clone()), server_username: None, server_password: None, diff --git a/src/client.rs b/src/client.rs index 23392b73a..405d72bee 100644 --- a/src/client.rs +++ b/src/client.rs @@ -14,7 +14,9 @@ use tokio::sync::mpsc::Sender; use crate::admin::{generate_server_parameters_for_admin, handle_admin}; use crate::auth_passthrough::refetch_auth_hash; -use crate::config::{get_config, get_idle_client_in_transaction_timeout, Address, PoolMode}; +use crate::config::{ + get_config, get_idle_client_in_transaction_timeout, Address, AuthType, PoolMode, +}; use crate::constants::*; use crate::messages::*; use crate::plugins::PluginOutput; @@ -463,8 +465,8 @@ where .count() == 1; - // Kick any client that's not admin while we're in admin-only mode. if !admin && admin_only { + // Kick any client that's not admin while we're in admin-only mode. debug!( "Rejecting non-admin connection to {} when in admin only mode", pool_name @@ -481,72 +483,76 @@ where let process_id: i32 = rand::random(); let secret_key: i32 = rand::random(); - // Perform MD5 authentication. - // TODO: Add SASL support. - let salt = md5_challenge(&mut write).await?; - - let code = match read.read_u8().await { - Ok(p) => p, - Err(_) => { - return Err(Error::ClientSocketError( - "password code".into(), - client_identifier, - )) - } - }; - - // PasswordMessage - if code as char != 'p' { - return Err(Error::ProtocolSyncError(format!( - "Expected p, got {}", - code as char - ))); - } - - let len = match read.read_i32().await { - Ok(len) => len, - Err(_) => { - return Err(Error::ClientSocketError( - "password message length".into(), - client_identifier, - )) - } - }; - - let mut password_response = vec![0u8; (len - 4) as usize]; - - match read.read_exact(&mut password_response).await { - Ok(_) => (), - Err(_) => { - return Err(Error::ClientSocketError( - "password message".into(), - client_identifier, - )) - } - }; - let mut prepared_statements_enabled = false; // Authenticate admin user. let (transaction_mode, mut server_parameters) = if admin { let config = get_config(); + // TODO: Add SASL support. + // Perform MD5 authentication. + match config.general.admin_auth_type { + AuthType::Trust => (), + AuthType::MD5 => { + let salt = md5_challenge(&mut write).await?; + + let code = match read.read_u8().await { + Ok(p) => p, + Err(_) => { + return Err(Error::ClientSocketError( + "password code".into(), + client_identifier, + )) + } + }; + + // PasswordMessage + if code as char != 'p' { + return Err(Error::ProtocolSyncError(format!( + "Expected p, got {}", + code as char + ))); + } - // Compare server and client hashes. - let password_hash = md5_hash_password( - &config.general.admin_username, - &config.general.admin_password, - &salt, - ); + let len = match read.read_i32().await { + Ok(len) => len, + Err(_) => { + return Err(Error::ClientSocketError( + "password message length".into(), + client_identifier, + )) + } + }; - if password_hash != password_response { - let error = Error::ClientGeneralError("Invalid password".into(), client_identifier); + let mut password_response = vec![0u8; (len - 4) as usize]; - warn!("{}", error); - wrong_password(&mut write, username).await?; + match read.read_exact(&mut password_response).await { + Ok(_) => (), + Err(_) => { + return Err(Error::ClientSocketError( + "password message".into(), + client_identifier, + )) + } + }; - return Err(error); - } + // Compare server and client hashes. + let password_hash = md5_hash_password( + &config.general.admin_username, + &config.general.admin_password, + &salt, + ); + + if password_hash != password_response { + let error = + Error::ClientGeneralError("Invalid password".into(), client_identifier); + warn!("{}", error); + wrong_password(&mut write, username).await?; + + return Err(error); + } + } + } (false, generate_server_parameters_for_admin()) } // Authenticate normal user. @@ -573,92 +579,143 @@ where // Obtain the hash to compare, we give preference to that written in cleartext in config // if there is nothing set in cleartext and auth passthrough (auth_query) is configured, we use the hash obtained // when the pool was created. If there is no hash there, we try to fetch it one more time. - let password_hash = if let Some(password) = &pool.settings.user.password { - Some(md5_hash_password(username, password, &salt)) - } else { - if !get_config().is_auth_query_configured() { - wrong_password(&mut write, username).await?; - return Err(Error::ClientAuthImpossible(username.into())); - } - - let mut hash = (*pool.auth_hash.read()).clone(); - - if hash.is_none() { - warn!( - "Query auth configured \ - but no hash password found \ - for pool {}. Will try to refetch it.", - pool_name - ); + match pool.settings.user.auth_type { + AuthType::Trust => (), + AuthType::MD5 => { + // Perform MD5 authentication. + // TODO: Add SASL support. + let salt = md5_challenge(&mut write).await?; + + let code = match read.read_u8().await { + Ok(p) => p, + Err(_) => { + return Err(Error::ClientSocketError( + "password code".into(), + client_identifier, + )) + } + }; + + // PasswordMessage + if code as char != 'p' { + return Err(Error::ProtocolSyncError(format!( + "Expected p, got {}", + code as char + ))); + } - match refetch_auth_hash(&pool).await { - Ok(fetched_hash) => { - warn!("Password for {}, obtained. Updating.", client_identifier); + let len = match read.read_i32().await { + Ok(len) => len, + Err(_) => { + return Err(Error::ClientSocketError( + "password message length".into(), + client_identifier, + )) + } + }; - { - let mut pool_auth_hash = pool.auth_hash.write(); - *pool_auth_hash = Some(fetched_hash.clone()); - } + let mut password_response = vec![0u8; (len - 4) as usize]; - hash = Some(fetched_hash); + match read.read_exact(&mut password_response).await { + Ok(_) => (), + Err(_) => { + return Err(Error::ClientSocketError( + "password message".into(), + client_identifier, + )) } + }; - Err(err) => { + let password_hash = if let Some(password) = &pool.settings.user.password { + Some(md5_hash_password(username, password, &salt)) + } else { + if !get_config().is_auth_query_configured() { wrong_password(&mut write, username).await?; - - return Err(Error::ClientAuthPassthroughError( - err.to_string(), - client_identifier, - )); + return Err(Error::ClientAuthImpossible(username.into())); } - } - }; - Some(md5_hash_second_pass(&hash.unwrap(), &salt)) - }; + let mut hash = (*pool.auth_hash.read()).clone(); - // Once we have the resulting hash, we compare with what the client gave us. - // If they do not match and auth query is set up, we try to refetch the hash one more time - // to see if the password has changed since the pool was created. - // - // @TODO: we could end up fetching again the same password twice (see above). - if password_hash.unwrap() != password_response { - warn!( - "Invalid password {}, will try to refetch it.", - client_identifier - ); + if hash.is_none() { + warn!( + "Query auth configured \ + but no hash password found \ + for pool {}. Will try to refetch it.", + pool_name + ); - let fetched_hash = match refetch_auth_hash(&pool).await { - Ok(fetched_hash) => fetched_hash, - Err(err) => { - wrong_password(&mut write, username).await?; + match refetch_auth_hash(&pool).await { + Ok(fetched_hash) => { + warn!( + "Password for {}, obtained. Updating.", + client_identifier + ); - return Err(err); - } - }; + { + let mut pool_auth_hash = pool.auth_hash.write(); + *pool_auth_hash = Some(fetched_hash.clone()); + } - let new_password_hash = md5_hash_second_pass(&fetched_hash, &salt); + hash = Some(fetched_hash); + } - // Ok password changed in server an auth is possible. - if new_password_hash == password_response { - warn!( - "Password for {}, changed in server. Updating.", - client_identifier - ); + Err(err) => { + wrong_password(&mut write, username).await?; - { - let mut pool_auth_hash = pool.auth_hash.write(); - *pool_auth_hash = Some(fetched_hash); + return Err(Error::ClientAuthPassthroughError( + err.to_string(), + client_identifier, + )); + } + } + }; + + Some(md5_hash_second_pass(&hash.unwrap(), &salt)) + }; + + // Once we have the resulting hash, we compare with what the client gave us. + // If they do not match and auth query is set up, we try to refetch the hash one more time + // to see if the password has changed since the pool was created. + // + // @TODO: we could end up fetching again the same password twice (see above). + if password_hash.unwrap() != password_response { + warn!( + "Invalid password {}, will try to refetch it.", + client_identifier + ); + + let fetched_hash = match refetch_auth_hash(&pool).await { + Ok(fetched_hash) => fetched_hash, + Err(err) => { + wrong_password(&mut write, username).await?; + + return Err(err); + } + }; + + let new_password_hash = md5_hash_second_pass(&fetched_hash, &salt); + + // Ok password changed in server an auth is possible. + if new_password_hash == password_response { + warn!( + "Password for {}, changed in server. Updating.", + client_identifier + ); + + { + let mut pool_auth_hash = pool.auth_hash.write(); + *pool_auth_hash = Some(fetched_hash); + } + } else { + wrong_password(&mut write, username).await?; + return Err(Error::ClientGeneralError( + "Invalid password".into(), + client_identifier, + )); + } } - } else { - wrong_password(&mut write, username).await?; - return Err(Error::ClientGeneralError( - "Invalid password".into(), - client_identifier, - )); } } - let transaction_mode = pool.settings.pool_mode == PoolMode::Transaction; prepared_statements_enabled = transaction_mode && pool.prepared_statement_cache.is_some(); diff --git a/src/config.rs b/src/config.rs index c7aaf4c35..b0d98fb52 100644 --- a/src/config.rs +++ b/src/config.rs @@ -208,6 +208,9 @@ impl Address { pub struct User { pub username: String, pub password: Option, + + #[serde(default = "User::default_auth_type")] + pub auth_type: AuthType, pub server_username: Option, pub server_password: Option, pub pool_size: u32, @@ -225,6 +228,7 @@ impl Default for User { User { username: String::from("postgres"), password: None, + auth_type: AuthType::MD5, server_username: None, server_password: None, pool_size: 15, @@ -239,6 +243,10 @@ impl Default for User { } impl User { + pub fn default_auth_type() -> AuthType { + AuthType::MD5 + } + fn validate(&self) -> Result<(), Error> { if let Some(min_pool_size) = self.min_pool_size { if min_pool_size > self.pool_size { @@ -334,6 +342,9 @@ pub struct General { pub admin_username: String, pub admin_password: String, + #[serde(default = "General::default_admin_auth_type")] + pub admin_auth_type: AuthType, + #[serde(default = "General::default_validate_config")] pub validate_config: bool, @@ -348,6 +359,10 @@ impl General { "0.0.0.0".into() } + pub fn default_admin_auth_type() -> AuthType { + AuthType::MD5 + } + pub fn default_port() -> u16 { 5432 } @@ -456,6 +471,7 @@ impl Default for General { verify_server_certificate: false, admin_username: String::from("admin"), admin_password: String::from("admin"), + admin_auth_type: AuthType::MD5, validate_config: true, auth_query: None, auth_query_user: None, @@ -476,6 +492,15 @@ pub enum PoolMode { Session, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Copy, Hash)] +pub enum AuthType { + #[serde(alias = "trust", alias = "Trust")] + Trust, + + #[serde(alias = "md5", alias = "MD5")] + MD5, +} + impl std::fmt::Display for PoolMode { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/tests/python/conftest.py b/tests/python/conftest.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/python/test_auth.py b/tests/python/test_auth.py new file mode 100644 index 000000000..bd9434292 --- /dev/null +++ b/tests/python/test_auth.py @@ -0,0 +1,71 @@ +import utils +import signal + +class TestTrustAuth: + @classmethod + def setup_method(cls): + config= """ + [general] + host = "0.0.0.0" + port = 6432 + admin_username = "admin_user" + admin_password = "" + admin_auth_type = "trust" + + [pools.sharded_db.users.0] + username = "sharding_user" + password = "sharding_user" + auth_type = "trust" + pool_size = 10 + min_pool_size = 1 + pool_mode = "transaction" + + [pools.sharded_db.shards.0] + servers = [ + [ "127.0.0.1", 5432, "primary" ], + ] + database = "shard0" + """ + utils.pgcat_generic_start(config) + + @classmethod + def teardown_method(self): + utils.pg_cat_send_signal(signal.SIGTERM) + + def test_admin_trust_auth(self): + conn, cur = utils.connect_db_trust(admin=True) + cur.execute("SHOW POOLS") + res = cur.fetchall() + print(res) + utils.cleanup_conn(conn, cur) + + def test_normal_trust_auth(self): + conn, cur = utils.connect_db_trust(autocommit=False) + cur.execute("SELECT 1") + res = cur.fetchall() + print(res) + utils.cleanup_conn(conn, cur) + +class TestMD5Auth: + @classmethod + def setup_method(cls): + utils.pgcat_start() + + @classmethod + def teardown_method(self): + utils.pg_cat_send_signal(signal.SIGTERM) + + def test_normal_db_access(self): + conn, cur = utils.connect_db(autocommit=False) + cur.execute("SELECT 1") + res = cur.fetchall() + print(res) + utils.cleanup_conn(conn, cur) + + def test_admin_db_access(self): + conn, cur = utils.connect_db(admin=True) + + cur.execute("SHOW POOLS") + res = cur.fetchall() + print(res) + utils.cleanup_conn(conn, cur) diff --git a/tests/python/test_pgcat.py b/tests/python/test_pgcat.py index dc2f11e5f..773715d4a 100644 --- a/tests/python/test_pgcat.py +++ b/tests/python/test_pgcat.py @@ -1,30 +1,12 @@ -import os + import signal import time import psycopg2 - import utils SHUTDOWN_TIMEOUT = 5 -def test_normal_db_access(): - utils.pgcat_start() - conn, cur = utils.connect_db(autocommit=False) - cur.execute("SELECT 1") - res = cur.fetchall() - print(res) - utils.cleanup_conn(conn, cur) - - -def test_admin_db_access(): - conn, cur = utils.connect_db(admin=True) - - cur.execute("SHOW POOLS") - res = cur.fetchall() - print(res) - utils.cleanup_conn(conn, cur) - def test_shutdown_logic(): @@ -256,3 +238,5 @@ def test_shutdown_logic(): utils.cleanup_conn(conn, cur) utils.pg_cat_send_signal(signal.SIGTERM) + + # - - - - - - - - - - - - - - - - - - diff --git a/tests/python/utils.py b/tests/python/utils.py index 5c49bce9c..9a1c6de97 100644 --- a/tests/python/utils.py +++ b/tests/python/utils.py @@ -1,20 +1,49 @@ -from typing import Tuple import os -import psutil import signal import time +from typing import Tuple +import tempfile +import psutil import psycopg2 PGCAT_HOST = "127.0.0.1" PGCAT_PORT = "6432" -def pgcat_start(): + +def _pgcat_start(config_path: str): pg_cat_send_signal(signal.SIGTERM) - os.system("./target/debug/pgcat .circleci/pgcat.toml &") + os.system(f"./target/debug/pgcat {config_path} &") time.sleep(2) +def pgcat_start(): + _pgcat_start(config_path='.circleci/pgcat.toml') + + +def pgcat_generic_start(config: str): + tmp = tempfile.NamedTemporaryFile() + with open(tmp.name, 'w') as f: + f.write(config) + _pgcat_start(config_path=tmp.name) + + +def glauth_send_signal(signal: signal.Signals): + try: + for proc in psutil.process_iter(["pid", "name"]): + if proc.name() == "glauth": + os.kill(proc.pid, signal) + except Exception as e: + # The process can be gone when we send this signal + print(e) + + if signal == signal.SIGTERM: + # Returns 0 if pgcat process exists + time.sleep(2) + if not os.system('pgrep glauth'): + raise Exception("glauth not closed after SIGTERM") + + def pg_cat_send_signal(signal: signal.Signals): try: for proc in psutil.process_iter(["pid", "name"]): @@ -54,6 +83,27 @@ def connect_db( return (conn, cur) +def connect_db_trust( + autocommit: bool = True, + admin: bool = False, +) -> Tuple[psycopg2.extensions.connection, psycopg2.extensions.cursor]: + + if admin: + user = "admin_user" + db = "pgcat" + else: + user = "sharding_user" + db = "sharded_db" + + conn = psycopg2.connect( + f"postgres://{user}@{PGCAT_HOST}:{PGCAT_PORT}/{db}?application_name=testing_pgcat", + connect_timeout=2, + ) + conn.autocommit = autocommit + cur = conn.cursor() + + return (conn, cur) + def cleanup_conn(conn: psycopg2.extensions.connection, cur: psycopg2.extensions.cursor): cur.close() From 2ad6e1bd3d1be47e9b4c85f9a17ce63261adf11e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 19:19:30 -0500 Subject: [PATCH 522/551] chore(deps): bump rexml from 3.2.8 to 3.3.6 in /tests/ruby (#803) Bumps [rexml](https://github.com/ruby/rexml) from 3.2.8 to 3.3.6. - [Release notes](https://github.com/ruby/rexml/releases) - [Changelog](https://github.com/ruby/rexml/blob/master/NEWS.md) - [Commits](https://github.com/ruby/rexml/compare/v3.2.8...v3.3.6) --- updated-dependencies: - dependency-name: rexml dependency-type: indirect ... Signed-off-by: dependabot[bot] --- tests/ruby/Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock index 048683b14..1a883ac63 100644 --- a/tests/ruby/Gemfile.lock +++ b/tests/ruby/Gemfile.lock @@ -24,8 +24,8 @@ GEM pg (1.3.2) rainbow (3.1.1) regexp_parser (2.3.1) - rexml (3.2.8) - strscan (>= 3.0.9) + rexml (3.3.6) + strscan rspec (3.11.0) rspec-core (~> 3.11.0) rspec-expectations (~> 3.11.0) From 4899a201241d1642a840c52ce64613dbee19c678 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 19:41:25 -0500 Subject: [PATCH 523/551] chore(deps): bump helm/chart-releaser-action from 1.5.0 to 1.6.0 (#812) Bumps [helm/chart-releaser-action](https://github.com/helm/chart-releaser-action) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/helm/chart-releaser-action/releases) - [Commits](https://github.com/helm/chart-releaser-action/compare/be16258da8010256c6e82849661221415f031968...a917fd15b20e8b64b94d9158ad54cd6345335584) --- updated-dependencies: - dependency-name: helm/chart-releaser-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/chart-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/chart-release.yaml b/.github/workflows/chart-release.yaml index 6399d4b45..fa5727247 100644 --- a/.github/workflows/chart-release.yaml +++ b/.github/workflows/chart-release.yaml @@ -32,7 +32,7 @@ jobs: version: v3.13.0 - name: Run chart-releaser - uses: helm/chart-releaser-action@be16258da8010256c6e82849661221415f031968 # v1.5.0 + uses: helm/chart-releaser-action@a917fd15b20e8b64b94d9158ad54cd6345335584 # v1.6.0 with: charts_dir: charts config: cr.yaml From 52d77e7a4b47aace6541e3508f7d0f122cbb4c47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 19:43:26 -0500 Subject: [PATCH 524/551] chore(deps): bump activesupport from 7.0.4.1 to 7.0.7.1 in /tests/ruby (#804) Bumps [activesupport](https://github.com/rails/rails) from 7.0.4.1 to 7.0.7.1. - [Release notes](https://github.com/rails/rails/releases) - [Changelog](https://github.com/rails/rails/blob/v7.2.1/activesupport/CHANGELOG.md) - [Commits](https://github.com/rails/rails/compare/v7.0.4.1...v7.0.7.1) --- updated-dependencies: - dependency-name: activesupport dependency-type: indirect ... Signed-off-by: dependabot[bot] --- tests/ruby/Gemfile.lock | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock index 1a883ac63..f47116596 100644 --- a/tests/ruby/Gemfile.lock +++ b/tests/ruby/Gemfile.lock @@ -1,22 +1,18 @@ GEM remote: https://rubygems.org/ specs: - activemodel (7.0.4.1) - activesupport (= 7.0.4.1) - activerecord (7.0.4.1) - activemodel (= 7.0.4.1) - activesupport (= 7.0.4.1) - activesupport (7.0.4.1) + activerecord (3.2.14) + activesupport (7.0.7.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) ast (2.4.2) - concurrent-ruby (1.1.10) + concurrent-ruby (1.3.4) diff-lcs (1.5.0) - i18n (1.12.0) + i18n (1.14.5) concurrent-ruby (~> 1.0) - minitest (5.17.0) + minitest (5.25.1) parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) @@ -55,7 +51,7 @@ GEM toml (0.3.0) parslet (>= 1.8.0, < 3.0.0) toxiproxy (2.0.1) - tzinfo (2.0.5) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) unicode-display_width (2.1.0) From 4c3627f8cba3e4d233b2e259b1454a3638fde044 Mon Sep 17 00:00:00 2001 From: Mostafa Abdelraouf Date: Fri, 13 Sep 2024 20:02:38 -0500 Subject: [PATCH 525/551] Fix Ruby tests (#819) Build is failing with this error Downloading activerecord-3.2.14 revealed dependencies not in the API or the lockfile (activesupport (= 3.2.14), activemodel (= 3.2.14), arel (~> 3.0.2), tzinfo (~> 0.3.29)). Either installing with `--full-index` or running `bundle update activerecord` should fix the problem. After ActiveSupport was updated. This PR fixes that --- tests/ruby/Gemfile.lock | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/ruby/Gemfile.lock b/tests/ruby/Gemfile.lock index f47116596..ba6999baf 100644 --- a/tests/ruby/Gemfile.lock +++ b/tests/ruby/Gemfile.lock @@ -1,18 +1,33 @@ GEM remote: https://rubygems.org/ specs: - activerecord (3.2.14) - activesupport (7.0.7.1) + activemodel (7.1.4) + activesupport (= 7.1.4) + activerecord (7.1.4) + activemodel (= 7.1.4) + activesupport (= 7.1.4) + timeout (>= 0.4.0) + activesupport (7.1.4) + base64 + bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) + connection_pool (>= 2.2.5) + drb i18n (>= 1.6, < 2) minitest (>= 5.1) + mutex_m tzinfo (~> 2.0) ast (2.4.2) + base64 (0.2.0) + bigdecimal (3.1.8) concurrent-ruby (1.3.4) + connection_pool (2.4.1) diff-lcs (1.5.0) + drb (2.2.1) i18n (1.14.5) concurrent-ruby (~> 1.0) minitest (5.25.1) + mutex_m (0.2.0) parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) @@ -48,6 +63,7 @@ GEM parser (>= 3.1.1.0) ruby-progressbar (1.11.0) strscan (3.1.0) + timeout (0.4.1) toml (0.3.0) parslet (>= 1.8.0, < 3.0.0) toxiproxy (2.0.1) From 45d7c8082f66112592afb68c61ad01368bbe3a3a Mon Sep 17 00:00:00 2001 From: Shijun Wang <2675460+Speissi@users.noreply.github.com> Date: Sat, 14 Sep 2024 17:57:17 +0300 Subject: [PATCH 526/551] Helm chart improvements including allowing user password to be pulled from K8s secret (#753) * Make user min_pool_size configurable * Set user server_lifetime only if specified * Increment chart version * Use default instea of or * Allow enabling server_tls * statement_timeout default value * Allow pulling password from existing secret --------- --- charts/pgcat/Chart.yaml | 2 +- charts/pgcat/templates/secret.yaml | 17 ++++++++++++++--- charts/pgcat/values.yaml | 7 ++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/charts/pgcat/Chart.yaml b/charts/pgcat/Chart.yaml index e05b239f9..49df3e111 100644 --- a/charts/pgcat/Chart.yaml +++ b/charts/pgcat/Chart.yaml @@ -5,4 +5,4 @@ maintainers: - name: Wildcard email: support@w6d.io appVersion: "1.2.0" -version: 0.2.0 +version: 0.2.1 diff --git a/charts/pgcat/templates/secret.yaml b/charts/pgcat/templates/secret.yaml index 05711ea99..358f4e97b 100644 --- a/charts/pgcat/templates/secret.yaml +++ b/charts/pgcat/templates/secret.yaml @@ -15,6 +15,7 @@ stringData: connect_timeout = {{ .Values.configuration.general.connect_timeout }} idle_timeout = {{ .Values.configuration.general.idle_timeout | int }} server_lifetime = {{ .Values.configuration.general.server_lifetime | int }} + server_tls = {{ .Values.configuration.general.server_tls }} idle_client_in_transaction_timeout = {{ .Values.configuration.general.idle_client_in_transaction_timeout | int }} healthcheck_timeout = {{ .Values.configuration.general.healthcheck_timeout }} healthcheck_delay = {{ .Values.configuration.general.healthcheck_delay }} @@ -58,11 +59,21 @@ stringData: ## [pools.{{ $pool.name | quote }}.users.{{ $index }}] username = {{ $user.username | quote }} + {{- if $user.password }} password = {{ $user.password | quote }} + {{- else if and $user.passwordSecret.name $user.passwordSecret.key }} + {{- $secret := (lookup "v1" "Secret" $.Release.Namespace $user.passwordSecret.name) }} + {{- if $secret }} + {{- $password := index $secret.data $user.passwordSecret.key | b64dec }} + password = {{ $password | quote }} + {{- end }} + {{- end }} pool_size = {{ $user.pool_size }} - statement_timeout = {{ $user.statement_timeout }} - min_pool_size = 3 - server_lifetime = 60000 + statement_timeout = {{ default 0 $user.statement_timeout }} + min_pool_size = {{ default 3 $user.min_pool_size }} + {{- if $user.server_lifetime }} + server_lifetime = {{ $user.server_lifetime }} + {{- end }} {{- if and $user.server_username $user.server_password }} server_username = {{ $user.server_username | quote }} server_password = {{ $user.server_password | quote }} diff --git a/charts/pgcat/values.yaml b/charts/pgcat/values.yaml index e87c576b7..20a4e27b8 100644 --- a/charts/pgcat/values.yaml +++ b/charts/pgcat/values.yaml @@ -175,6 +175,9 @@ configuration: # Max connection lifetime before it's closed, even if actively used. server_lifetime: 86400000 # 24 hours + # Whether to use TLS for server connections or not. + server_tls: false + # How long a client is allowed to be idle while in a transaction (ms). idle_client_in_transaction_timeout: 0 # milliseconds @@ -315,7 +318,9 @@ configuration: # ## Credentials for users that may connect to this cluster # ## @param users [array] # ## @param users[0].username Name of the env var (required) - # ## @param users[0].password Value for the env var (required) + # ## @param users[0].password Value for the env var (required) leave empty to use existing secret see passwordSecret.name and passwordSecret.key + # ## @param users[0].passwordSecret.name Name of the secret containing the password + # ## @param users[0].passwordSecret.key Key in the secret containing the password # ## @param users[0].pool_size Maximum number of server connections that can be established for this user # ## @param users[0].statement_timeout Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. # users: [] From a77b27b74f80e1b4c0dd17910bf357d28442de00 Mon Sep 17 00:00:00 2001 From: Sean McGivern <27fv8yygye@snkmail.com> Date: Wed, 18 Sep 2024 12:56:17 +0100 Subject: [PATCH 527/551] Improve documentation for connect_timeout and add min_pool_size (#822) Currently, `connect_timeout` sounds like it should be for connections to the Postgres server. It's actually used for obtaining a connection from the pool. --- CONFIG.md | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 7635bf6ba..4e984edef 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -36,10 +36,11 @@ Port at which prometheus exporter listens on. ### connect_timeout ``` path: general.connect_timeout -default: 5000 # milliseconds +default: 1000 # milliseconds ``` -How long to wait before aborting a server connection (ms). +How long the client waits to obtain a server connection before aborting (ms). +This is similar to PgBouncer's `query_wait_timeout`. ### idle_timeout ``` @@ -462,10 +463,18 @@ path: pools..users..pool_size default: 9 ``` -Maximum number of server connections that can be established for this user +Maximum number of server connections that can be established for this user. The maximum number of connection from a single Pgcat process to any database in the cluster is the sum of pool_size across all users. +### min_pool_size +``` +path: pools..users..min_pool_size +default: 0 +``` + +Minimum number of idle server connections to retain for this pool. + ### statement_timeout ``` path: pools..users..statement_timeout @@ -475,6 +484,16 @@ default: 0 Maximum query duration. Dangerous, but protects against DBs that died in a non-obvious way. 0 means it is disabled. +### connect_timeout +``` +path: pools..users..connect_timeout +default: # milliseconds +``` + +How long the client waits to obtain a server connection before aborting (ms). +This is similar to PgBouncer's `query_wait_timeout`. +If unset, uses the `connect_timeout` defined globally. + ## `pools..shards.` Section ### servers @@ -502,4 +521,3 @@ default: "shard0" ``` Database name (e.g. "postgres") - From c9001a812782be413ccacfe3a48e9c6456bf2aff Mon Sep 17 00:00:00 2001 From: Sebastian Serth Date: Mon, 23 Sep 2024 00:14:04 +0200 Subject: [PATCH 528/551] End Prometheus stats with a new line separator (#826) End prometheus stats with a new line separator According to the [OpenMetrics specification](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#overall-structure), each line MUST end with `\n`. Previously, the last line was not ending with `\n`, so that strict parsers had issues reading the Prometheus stats. --- src/prometheus.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/prometheus.rs b/src/prometheus.rs index 93348b853..6c9a1be8c 100644 --- a/src/prometheus.rs +++ b/src/prometheus.rs @@ -309,6 +309,7 @@ async fn prometheus_stats( push_pool_stats(&mut lines); push_server_stats(&mut lines); push_database_stats(&mut lines); + lines.push("".to_string()); // Ensure to end the stats with a line terminator as required by the specification. Response::builder() .header("content-type", "text/plain; version=0.0.4") From 6c794a86ff417a95cd62727fa9e55c169a0c6d89 Mon Sep 17 00:00:00 2001 From: Javier Goday Date: Wed, 23 Oct 2024 13:20:04 +0200 Subject: [PATCH 529/551] #829: read/write splitting on CTE mutable statements (#835) --- src/query_router.rs | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/query_router.rs b/src/query_router.rs index bc6ed2c66..939abee1f 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -386,6 +386,18 @@ impl QueryRouter { } } + /// Determines if a query is mutable or not. + fn query_is_mutable_statement(q: &sqlparser::ast::Query) -> bool { + use sqlparser::ast::*; + + match q.body.as_ref() { + SetExpr::Insert(_) => true, + SetExpr::Update(_) => true, + SetExpr::Query(q) => Self::query_is_mutable_statement(q), + _ => false, + } + } + /// Try to infer which server to connect to based on the contents of the query. pub fn infer(&mut self, ast: &Vec) -> Result<(), Error> { if !self.pool_settings.query_parser_read_write_splitting { @@ -428,8 +440,9 @@ impl QueryRouter { }; let has_locks = !query.locks.is_empty(); + let is_mutable_statement = Self::query_is_mutable_statement(query); - if has_locks { + if has_locks || is_mutable_statement { self.active_role = Some(Role::Primary); } else if !visited_write_statement { // If we already visited a write statement, we should be going to the primary. @@ -1113,6 +1126,26 @@ mod test { assert_eq!(qr.role(), None); } + #[test] + fn test_split_cte_queries() { + QueryRouter::setup(); + let mut qr = QueryRouter::new(); + qr.pool_settings.query_parser_read_write_splitting = true; + qr.pool_settings.query_parser_enabled = true; + + let query = simple_query( + "WITH t AS ( + SELECT id FROM users WHERE name ILIKE '%ja%' + ) + UPDATE user_languages + SET settings = '{}' + FROM t WHERE t.id = user_id;", + ); + let ast = qr.parse(&query).unwrap(); + assert!(qr.infer(&ast).is_ok()); + assert_eq!(qr.role(), Some(Role::Primary)); + } + #[test] fn test_infer_replica() { QueryRouter::setup(); From 275ad30581547038694c985bd46bceca0aec16ae Mon Sep 17 00:00:00 2001 From: Mostafa Date: Wed, 23 Oct 2024 06:38:07 -0500 Subject: [PATCH 530/551] Rename a couple of variables (#839) --- src/query_router.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/query_router.rs b/src/query_router.rs index 939abee1f..c2548bbec 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -386,14 +386,14 @@ impl QueryRouter { } } - /// Determines if a query is mutable or not. - fn query_is_mutable_statement(q: &sqlparser::ast::Query) -> bool { + /// Determines if a query is a mutation or not. + fn is_mutation_query(q: &sqlparser::ast::Query) -> bool { use sqlparser::ast::*; match q.body.as_ref() { SetExpr::Insert(_) => true, SetExpr::Update(_) => true, - SetExpr::Query(q) => Self::query_is_mutable_statement(q), + SetExpr::Query(q) => Self::is_mutation_query(q), _ => false, } } @@ -440,9 +440,9 @@ impl QueryRouter { }; let has_locks = !query.locks.is_empty(); - let is_mutable_statement = Self::query_is_mutable_statement(query); + let has_mutation = Self::is_mutation_query(query); - if has_locks || is_mutable_statement { + if has_locks || has_mutation { self.active_role = Some(Role::Primary); } else if !visited_write_statement { // If we already visited a write statement, we should be going to the primary. From bc6161d6475c5e3dd6122c1df88a6b855be5a22e Mon Sep 17 00:00:00 2001 From: Nicolas Vanelslande Date: Mon, 28 Oct 2024 11:49:36 +0000 Subject: [PATCH 531/551] Bump bb8 from 0.8.1 to 0.8.6 (#709) * Update bb8 to 0.8.6 To get https://github.com/djc/bb8/pull/186 and https://github.com/djc/bb8/pull/189 which fix potential deadlocks (https://github.com/djc/bb8/issues/154). Also, this (https://github.com/djc/bb8/pull/225) was needed to prevent a connection leak which was conveniently spotted in our integration tests. * Ignore ./.bundle (created by dev console) --------- --- .gitignore | 1 + Cargo.lock | 5 ++--- Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index ae1348147..02d8751dc 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ dev/cache !dev/cache/.keepme .venv **/__pycache__ +.bundle \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index e2f898535..fb58c7b2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -192,12 +192,11 @@ checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bb8" -version = "0.8.1" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b4b0f25f18bcdc3ac72bdb486ed0acf7e185221fd4dc985bc15db5800b0ba2" +checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" dependencies = [ "async-trait", - "futures-channel", "futures-util", "parking_lot", "tokio", diff --git a/Cargo.toml b/Cargo.toml index f75e91858..21cfb0c34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" tokio = { version = "1", features = ["full"] } bytes = "1" md-5 = "0.10" -bb8 = "0.8.1" +bb8 = "=0.8.6" async-trait = "0.1" rand = "0.8" chrono = "0.4" From da71bd59f7c33ec9f5bbed7ef8da1bb57be550c4 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Sat, 2 Nov 2024 11:55:18 -0500 Subject: [PATCH 532/551] Attempt a helm chart release (#844) Attempt a release --- charts/pgcat/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/pgcat/Chart.yaml b/charts/pgcat/Chart.yaml index 49df3e111..cdf8a66a4 100644 --- a/charts/pgcat/Chart.yaml +++ b/charts/pgcat/Chart.yaml @@ -5,4 +5,4 @@ maintainers: - name: Wildcard email: support@w6d.io appVersion: "1.2.0" -version: 0.2.1 +version: 0.2.2 From 4684030845dac2db5f455b53bb2b30b3231b4b20 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Sat, 2 Nov 2024 18:05:41 -0500 Subject: [PATCH 533/551] Another no-op release for helm (#845) Another no-op release --- charts/pgcat/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/pgcat/Chart.yaml b/charts/pgcat/Chart.yaml index cdf8a66a4..3b660112b 100644 --- a/charts/pgcat/Chart.yaml +++ b/charts/pgcat/Chart.yaml @@ -5,4 +5,4 @@ maintainers: - name: Wildcard email: support@w6d.io appVersion: "1.2.0" -version: 0.2.2 +version: 0.2.3 From 8238ff069fb9ee5596320e01e3434d74f796f459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=AD=C5=A5a=20Tauer?= Date: Thu, 7 Nov 2024 13:17:09 +0100 Subject: [PATCH 534/551] Fixing invalid setting name in pgcat.toml (#849) --- pgcat.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgcat.toml b/pgcat.toml index 9e19c13b9..87f2700cf 100644 --- a/pgcat.toml +++ b/pgcat.toml @@ -179,7 +179,7 @@ primary_reads_enabled = true # `random`: picks a shard at random # `random_healthy`: picks a shard at random favoring shards with the least number of recent errors # `shard_`: e.g. shard_0, shard_4, etc. picks a specific shard, everytime -# no_shard_specified_behavior = "shard_0" +# default_shard = "shard_0" # So what if you wanted to implement a different hashing function, # or you've already built one and you want this pooler to use it? From 3d48265bd578eea6bdeaff7af9cf4cfd65f981be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Thu, 7 Nov 2024 18:11:11 +0100 Subject: [PATCH 535/551] Do not unban replicas if a primary is available (#843) Add `unban_replicas_when_all_banned` to control unbanning replicas behavior. --- CONFIG.md | 10 ++++++++++ README.md | 2 +- src/config.rs | 4 ++++ src/pool.rs | 10 +++++++++- src/query_router.rs | 2 ++ 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 4e984edef..3dde1e18b 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -130,6 +130,16 @@ default: 60 # seconds How long to ban a server if it fails a health check (seconds). +### unban_replicas_when_all_banned +``` +path: general.unban_replicas_when_all_banned +default: true +``` + +Whether or not we should unban all replicas when they are all banned. This is set +to true by default to prevent disconnection when we have replicas with a false positive +health check. + ### log_client_connections ``` path: general.log_client_connections diff --git a/README.md b/README.md index 21e6da701..41c9f9459 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ The setting will persist until it's changed again or the client disconnects. By default, all queries are routed to the first available server; `default_role` setting controls this behavior. ### Failover -All servers are checked with a `;` (very fast) query before being given to a client. Additionally, the server health is monitored with every client query that it processes. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the ban list is cleared: this is a safety precaution against false positives. The primary can never be banned. +All servers are checked with a `;` (very fast) query before being given to a client. Additionally, the server health is monitored with every client query that it processes. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the behavior is controlled by the configuration parameter `unban_replicas_when_all_banned`. If it is set to true (the default), the ban list is cleared: this is a safety precaution against false positives, if it is set to false, no replicas will be available until they become healthy. The primary can never be banned. The ban time can be changed with `ban_time`. The default is 60 seconds. diff --git a/src/config.rs b/src/config.rs index b0d98fb52..9b8cc072f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -315,6 +315,9 @@ pub struct General { #[serde(default = "General::default_ban_time")] pub ban_time: i64, + #[serde(default)] // True + pub unban_replicas_when_all_banned: bool, + #[serde(default = "General::default_idle_client_in_transaction_timeout")] pub idle_client_in_transaction_timeout: u64, @@ -460,6 +463,7 @@ impl Default for General { healthcheck_timeout: Self::default_healthcheck_timeout(), healthcheck_delay: Self::default_healthcheck_delay(), ban_time: Self::default_ban_time(), + unban_replicas_when_all_banned: true, idle_client_in_transaction_timeout: Self::default_idle_client_in_transaction_timeout(), server_lifetime: Self::default_server_lifetime(), server_round_robin: Self::default_server_round_robin(), diff --git a/src/pool.rs b/src/pool.rs index 7915a0a41..f9b481a20 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -189,6 +189,9 @@ pub struct PoolSettings { // Ban time pub ban_time: i64, + // Should we automatically unban replicas when all are banned? + pub unban_replicas_when_all_banned: bool, + // Regex for searching for the sharding key in SQL statements pub sharding_key_regex: Option, @@ -228,6 +231,7 @@ impl Default for PoolSettings { healthcheck_delay: General::default_healthcheck_delay(), healthcheck_timeout: General::default_healthcheck_timeout(), ban_time: General::default_ban_time(), + unban_replicas_when_all_banned: true, sharding_key_regex: None, shard_id_regex: None, regex_search_limit: 1000, @@ -541,6 +545,9 @@ impl ConnectionPool { healthcheck_delay: config.general.healthcheck_delay, healthcheck_timeout: config.general.healthcheck_timeout, ban_time: config.general.ban_time, + unban_replicas_when_all_banned: config + .general + .unban_replicas_when_all_banned, sharding_key_regex: pool_config .sharding_key_regex .clone() @@ -946,8 +953,9 @@ impl ConnectionPool { let read_guard = self.banlist.read(); let all_replicas_banned = read_guard[address.shard].len() == replicas_available; drop(read_guard); + let unban_replicas_when_all_banned = self.settings.clone().unban_replicas_when_all_banned; - if all_replicas_banned { + if all_replicas_banned && unban_replicas_when_all_banned { let mut write_guard = self.banlist.write(); warn!("Unbanning all replicas."); write_guard[address.shard].clear(); diff --git a/src/query_router.rs b/src/query_router.rs index c2548bbec..cc502a0db 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1464,6 +1464,7 @@ mod test { healthcheck_delay: PoolSettings::default().healthcheck_delay, healthcheck_timeout: PoolSettings::default().healthcheck_timeout, ban_time: PoolSettings::default().ban_time, + unban_replicas_when_all_banned: true, sharding_key_regex: None, shard_id_regex: None, default_shard: crate::config::DefaultShard::Shard(0), @@ -1542,6 +1543,7 @@ mod test { healthcheck_delay: PoolSettings::default().healthcheck_delay, healthcheck_timeout: PoolSettings::default().healthcheck_timeout, ban_time: PoolSettings::default().ban_time, + unban_replicas_when_all_banned: true, sharding_key_regex: Some(Regex::new(r"/\* sharding_key: (\d+) \*/").unwrap()), shard_id_regex: Some(Regex::new(r"/\* shard_id: (\d+) \*/").unwrap()), default_shard: crate::config::DefaultShard::Shard(0), From 1704d54e6fb252f644d3274ea66992aafef1d659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Thu, 7 Nov 2024 18:11:49 +0100 Subject: [PATCH 536/551] Fix `default_role` being ignored when `query_parser_enabled` was false (#847) Fix default_role being ignored when query_parser_enabled was false --- src/client.rs | 1 + src/query_router.rs | 5 +++++ tests/ruby/load_balancing_spec.rb | 36 ++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index 405d72bee..c226436e6 100644 --- a/src/client.rs +++ b/src/client.rs @@ -881,6 +881,7 @@ where }; query_router.update_pool_settings(&pool.settings); + query_router.set_default_role(); // Our custom protocol loop. // We expect the client to either start a transaction with regular queries diff --git a/src/query_router.rs b/src/query_router.rs index cc502a0db..59cca232e 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1061,6 +1061,11 @@ impl QueryRouter { self.active_shard } + /// Set active_role as the default_role specified in the pool. + pub fn set_default_role(&mut self) { + self.active_role = self.pool_settings.default_role; + } + /// Get the current desired server role we should be talking to. pub fn role(&self) -> Option { self.active_role diff --git a/tests/ruby/load_balancing_spec.rb b/tests/ruby/load_balancing_spec.rb index 172e6069b..f00f8db1c 100644 --- a/tests/ruby/load_balancing_spec.rb +++ b/tests/ruby/load_balancing_spec.rb @@ -56,6 +56,41 @@ end end end + + context "when all replicas are down " do + let(:processes) { Helpers::Pgcat.single_shard_setup("sharded_db", 5, "transaction", "random", "debug", {"default_role" => "replica"}) } + + it "unbans them automatically to prevent false positives in health checks that could make all replicas unavailable" do + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + failed_count = 0 + number_of_replicas = processes[:replicas].length + + # Take down all replicas + processes[:replicas].each(&:take_down) + + (number_of_replicas + 1).times do |n| + conn.async_exec("SELECT 1 + 2") + rescue + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + failed_count += 1 + end + + expect(failed_count).to eq(number_of_replicas + 1) + failed_count = 0 + + # Ban_time is configured to 60 so this reset will only work + # if the replicas are unbanned automatically + processes[:replicas].each(&:reset) + + number_of_replicas.times do + conn.async_exec("SELECT 1 + 2") + rescue + conn = PG.connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + failed_count += 1 + end + expect(failed_count).to eq(0) + end + end end describe "Least Outstanding Queries Load Balancing" do @@ -161,4 +196,3 @@ end end end - From c60f4b0d13504f88d39e588ee2d09450e83a34ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Fern=C3=A1ndez?= Date: Thu, 7 Nov 2024 22:00:43 +0100 Subject: [PATCH 537/551] Revert "Do not unban replicas if a primary is available" (#850) Revert "Do not unban replicas if a primary is available (#843)" This reverts commit 3d48265bd578eea6bdeaff7af9cf4cfd65f981be. --- CONFIG.md | 10 ---------- README.md | 2 +- src/config.rs | 4 ---- src/pool.rs | 10 +--------- src/query_router.rs | 2 -- 5 files changed, 2 insertions(+), 26 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 3dde1e18b..4e984edef 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -130,16 +130,6 @@ default: 60 # seconds How long to ban a server if it fails a health check (seconds). -### unban_replicas_when_all_banned -``` -path: general.unban_replicas_when_all_banned -default: true -``` - -Whether or not we should unban all replicas when they are all banned. This is set -to true by default to prevent disconnection when we have replicas with a false positive -health check. - ### log_client_connections ``` path: general.log_client_connections diff --git a/README.md b/README.md index 41c9f9459..21e6da701 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ The setting will persist until it's changed again or the client disconnects. By default, all queries are routed to the first available server; `default_role` setting controls this behavior. ### Failover -All servers are checked with a `;` (very fast) query before being given to a client. Additionally, the server health is monitored with every client query that it processes. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the behavior is controlled by the configuration parameter `unban_replicas_when_all_banned`. If it is set to true (the default), the ban list is cleared: this is a safety precaution against false positives, if it is set to false, no replicas will be available until they become healthy. The primary can never be banned. +All servers are checked with a `;` (very fast) query before being given to a client. Additionally, the server health is monitored with every client query that it processes. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the ban list is cleared: this is a safety precaution against false positives. The primary can never be banned. The ban time can be changed with `ban_time`. The default is 60 seconds. diff --git a/src/config.rs b/src/config.rs index 9b8cc072f..b0d98fb52 100644 --- a/src/config.rs +++ b/src/config.rs @@ -315,9 +315,6 @@ pub struct General { #[serde(default = "General::default_ban_time")] pub ban_time: i64, - #[serde(default)] // True - pub unban_replicas_when_all_banned: bool, - #[serde(default = "General::default_idle_client_in_transaction_timeout")] pub idle_client_in_transaction_timeout: u64, @@ -463,7 +460,6 @@ impl Default for General { healthcheck_timeout: Self::default_healthcheck_timeout(), healthcheck_delay: Self::default_healthcheck_delay(), ban_time: Self::default_ban_time(), - unban_replicas_when_all_banned: true, idle_client_in_transaction_timeout: Self::default_idle_client_in_transaction_timeout(), server_lifetime: Self::default_server_lifetime(), server_round_robin: Self::default_server_round_robin(), diff --git a/src/pool.rs b/src/pool.rs index f9b481a20..7915a0a41 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -189,9 +189,6 @@ pub struct PoolSettings { // Ban time pub ban_time: i64, - // Should we automatically unban replicas when all are banned? - pub unban_replicas_when_all_banned: bool, - // Regex for searching for the sharding key in SQL statements pub sharding_key_regex: Option, @@ -231,7 +228,6 @@ impl Default for PoolSettings { healthcheck_delay: General::default_healthcheck_delay(), healthcheck_timeout: General::default_healthcheck_timeout(), ban_time: General::default_ban_time(), - unban_replicas_when_all_banned: true, sharding_key_regex: None, shard_id_regex: None, regex_search_limit: 1000, @@ -545,9 +541,6 @@ impl ConnectionPool { healthcheck_delay: config.general.healthcheck_delay, healthcheck_timeout: config.general.healthcheck_timeout, ban_time: config.general.ban_time, - unban_replicas_when_all_banned: config - .general - .unban_replicas_when_all_banned, sharding_key_regex: pool_config .sharding_key_regex .clone() @@ -953,9 +946,8 @@ impl ConnectionPool { let read_guard = self.banlist.read(); let all_replicas_banned = read_guard[address.shard].len() == replicas_available; drop(read_guard); - let unban_replicas_when_all_banned = self.settings.clone().unban_replicas_when_all_banned; - if all_replicas_banned && unban_replicas_when_all_banned { + if all_replicas_banned { let mut write_guard = self.banlist.write(); warn!("Unbanning all replicas."); write_guard[address.shard].clear(); diff --git a/src/query_router.rs b/src/query_router.rs index 59cca232e..2ed6b7557 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1469,7 +1469,6 @@ mod test { healthcheck_delay: PoolSettings::default().healthcheck_delay, healthcheck_timeout: PoolSettings::default().healthcheck_timeout, ban_time: PoolSettings::default().ban_time, - unban_replicas_when_all_banned: true, sharding_key_regex: None, shard_id_regex: None, default_shard: crate::config::DefaultShard::Shard(0), @@ -1548,7 +1547,6 @@ mod test { healthcheck_delay: PoolSettings::default().healthcheck_delay, healthcheck_timeout: PoolSettings::default().healthcheck_timeout, ban_time: PoolSettings::default().ban_time, - unban_replicas_when_all_banned: true, sharding_key_regex: Some(Regex::new(r"/\* sharding_key: (\d+) \*/").unwrap()), shard_id_regex: Some(Regex::new(r"/\* shard_id: (\d+) \*/").unwrap()), default_shard: crate::config::DefaultShard::Shard(0), From 5ecd89795c159eca7cd76a2ed04ba142a3f59cb5 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Fri, 8 Nov 2024 06:04:42 -0600 Subject: [PATCH 538/551] Use main branch for helm chart releases (#852) --- cr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cr.yaml b/cr.yaml index 0e3201dd6..ef1b81e0b 100644 --- a/cr.yaml +++ b/cr.yaml @@ -1 +1,2 @@ sign: false +pages_branch: main From 801da13c65c5ed1b5b571094e6c2ad2feaadaa33 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Fri, 8 Nov 2024 06:07:12 -0600 Subject: [PATCH 539/551] Another no-op helm release (#853) --- charts/pgcat/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/pgcat/Chart.yaml b/charts/pgcat/Chart.yaml index 3b660112b..0d020657e 100644 --- a/charts/pgcat/Chart.yaml +++ b/charts/pgcat/Chart.yaml @@ -5,4 +5,4 @@ maintainers: - name: Wildcard email: support@w6d.io appVersion: "1.2.0" -version: 0.2.3 +version: 0.2.4 From 74d738623bfd8b424935d83bf895e1696348b350 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Mon, 11 Nov 2024 09:24:17 -0600 Subject: [PATCH 540/551] Fix contact info for Helm chart (#861) * Fix contact info for Helm chart --- charts/pgcat/Chart.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/pgcat/Chart.yaml b/charts/pgcat/Chart.yaml index 0d020657e..430c839bf 100644 --- a/charts/pgcat/Chart.yaml +++ b/charts/pgcat/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: pgcat description: A Helm chart for PgCat a PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load balancing, failover and mirroring. maintainers: - - name: Wildcard - email: support@w6d.io + - name: PostgresML + email: team@postgresml.org appVersion: "1.2.0" -version: 0.2.4 +version: 0.2.5 From 66a04883aeac847c623673cffde768a7037475f5 Mon Sep 17 00:00:00 2001 From: Vitalii Tverdokhlib Date: Sun, 17 Nov 2024 14:51:05 +0200 Subject: [PATCH 541/551] DOCS: small typo LIMT (Update README.md) (#866) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 21e6da701..04c0a3954 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ User.find_by_email("test@example.com") ```sql -- Grab a bunch of users from shard 1 SET SHARD TO '1'; -SELECT * FROM users LIMT 10; +SELECT * FROM users LIMIT 10; -- Find by id SET SHARDING KEY TO '1234'; From 3f3a67e137de895241a12e1182c856d7fc6c9869 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sat, 23 Nov 2024 13:25:37 +0000 Subject: [PATCH 542/551] chore(deps): bump sqlparser from 0.41.0 to 0.52.0 (#870) * chore(deps): bump sqlparser from 0.41.0 to 0.52.0 Bumps [sqlparser](https://github.com/apache/datafusion-sqlparser-rs) from 0.41.0 to 0.52.0. - [Changelog](https://github.com/apache/datafusion-sqlparser-rs/blob/main/CHANGELOG.md) - [Commits](https://github.com/apache/datafusion-sqlparser-rs/commits) --- updated-dependencies: - dependency-name: sqlparser dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * bump * Update to latest sqlparser version --------- Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- src/query_router.rs | 54 ++++++++++++++++----------------------------- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb58c7b2e..0ff161b3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1525,9 +1525,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.41.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" +checksum = "9a875d8cd437cc8a97e9aeaeea352ec9a19aea99c23e9effb17757291de80b08" dependencies = [ "log", "sqlparser_derive", diff --git a/Cargo.toml b/Cargo.toml index 21cfb0c34..af15ab4f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ serde_derive = "1" regex = "1" num_cpus = "1" once_cell = "1" -sqlparser = { version = "0.41", features = ["visitor"] } +sqlparser = { version = "0.52", features = ["visitor"] } log = "0.4" arc-swap = "1" parking_lot = "0.12.1" diff --git a/src/query_router.rs b/src/query_router.rs index 2ed6b7557..3e485a0dc 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -504,55 +504,33 @@ impl QueryRouter { let mut table_names = Vec::new(); match q { - Insert { - or, - into: _, - table_name, - columns, - overwrite: _, - source, - partitioned, - after_columns, - table: _, - on: _, - returning: _, - ignore: _, - } => { + Insert(i) => { // Not supported in postgres. - assert!(or.is_none()); - assert!(partitioned.is_none()); - assert!(after_columns.is_empty()); + assert!(i.or.is_none()); + assert!(i.partitioned.is_none()); + assert!(i.after_columns.is_empty()); - Self::process_table(table_name, &mut table_names); - if let Some(source) = source { - Self::process_query(source, &mut exprs, &mut table_names, &Some(columns)); + Self::process_table(&i.table_name, &mut table_names); + if let Some(source) = &i.source { + Self::process_query(source, &mut exprs, &mut table_names, &Some(&i.columns)); } } - Delete { - tables, - from, - using, - selection, - returning: _, - order_by: _, - limit: _, - } => { - if let Some(expr) = selection { + Delete(d) => { + if let Some(expr) = &d.selection { exprs.push(expr.clone()); } // Multi tables delete are not supported in postgres. - assert!(tables.is_empty()); + assert!(d.tables.is_empty()); - Self::process_tables_with_join(from, &mut exprs, &mut table_names); - if let Some(using_tbl_with_join) = using { + if let Some(using_tbl_with_join) = &d.using { Self::process_tables_with_join( using_tbl_with_join, &mut exprs, &mut table_names, ); } - Self::process_selection(selection, &mut exprs); + Self::process_selection(&d.selection, &mut exprs); } Update { table, @@ -822,7 +800,13 @@ impl QueryRouter { for a in assignments { if sharding_key[0].value == "*" - && sharding_key[1].value == a.id.last().unwrap().value.to_lowercase() + && sharding_key[1].value + == a.target + .to_string() + .split('.') + .last() + .unwrap() + .to_lowercase() { return Err(Error::QueryRouterParserError( "Sharding key cannot be updated.".into(), From f7692c1dd918ca0defb4af351d9ed68a5dd17f4b Mon Sep 17 00:00:00 2001 From: Nadav Shatz Date: Sun, 22 Dec 2024 13:23:57 +0200 Subject: [PATCH 543/551] Add DB activity based routing (#864) --- CONFIG.md | 39 ++++ Cargo.lock | 289 +++++++++++++++++++++--- Cargo.toml | 6 +- charts/pgcat/Chart.yaml | 2 +- charts/pgcat/templates/secret.yaml | 4 + charts/pgcat/values.yaml | 16 ++ src/config.rs | 69 ++++++ src/pool.rs | 20 ++ src/query_router.rs | 346 ++++++++++++++++++++++++++++- 9 files changed, 755 insertions(+), 36 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 4e984edef..b6e16a7c8 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -309,6 +309,45 @@ If the client doesn't specify, PgCat routes traffic to this role by default. `replica` round-robin between replicas only without touching the primary, `primary` all queries go to the primary unless otherwise specified. +### db_activity_based_routing +``` +path: pools..db_activity_based_routing +default: false +``` + +If enabled, PgCat will route queries to the primary if the queried table was recently written to. +Only relevant when `query_parser_enabled` *and* `query_parser_read_write_splitting` is enabled. + +##### Considerations: +- *This feature is experimental and may not work as expected.* +- This feature only works when the same PgCat instance is used for both reads and writes to the database. +- This feature is not relevant when the primary is not part of the pool of databases used for load balancing of read queries. +- If more than one PgCat instance is used for HA purposes, this feature will not work as expected. A way to still make it work is by using sticky sessions. + +### db_activity_based_ms_init_delay +``` +path: pools..db_activity_based_ms_init_delay +default: 100 +``` + +The delay in milliseconds before the first activity-based routing check is performed. + +### db_activity_ttl +``` +path: pools..db_activity_ttl +default: 900 +``` + +The time in seconds after which a DB is considered inactive when no queries/updates are performed to it. + +### table_mutation_cache_ms_ttl +``` +path: pools..table_mutation_cache_ms_ttl +default: 50 +``` + +The time in milliseconds after a write to a table that all queries to that table will be routed to the primary. + ### prepared_statements_cache_size ``` path: general.prepared_statements_cache_size diff --git a/Cargo.lock b/Cargo.lock index 0ff161b3c..2f94c3908 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -132,7 +132,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -143,7 +143,7 @@ checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -229,6 +229,12 @@ version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + [[package]] name = "byteorder" version = "1.4.3" @@ -241,6 +247,37 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", +] + [[package]] name = "cc" version = "1.0.79" @@ -300,7 +337,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -330,6 +367,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crypto-common" version = "0.1.6" @@ -340,6 +392,19 @@ dependencies = [ "typenum", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.4.0" @@ -402,6 +467,15 @@ dependencies = [ "libc", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + [[package]] name = "exitcode" version = "1.1.2" @@ -414,6 +488,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fastrand" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + [[package]] name = "fnv" version = "1.0.7" @@ -485,7 +565,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -545,6 +625,12 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "h2" version = "0.4.6" @@ -918,6 +1004,21 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mini-moka" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c325dfab65f261f386debee8b0969da215b3fa0037e74c8a1234db7ba986d803" +dependencies = [ + "crossbeam-channel", + "crossbeam-utils", + "dashmap", + "skeptic", + "smallvec", + "tagptr", + "triomphe", +] + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -992,9 +1093,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "overload" @@ -1033,7 +1134,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pgcat" -version = "1.2.0" +version = "1.3.0" dependencies = [ "arc-swap", "async-trait", @@ -1055,6 +1156,7 @@ dependencies = [ "log", "lru", "md-5", + "mini-moka", "nix", "num_cpus", "once_cell", @@ -1069,6 +1171,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "serial_test", "sha-1", "sha2", "socket2 0.4.9", @@ -1114,7 +1217,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -1143,7 +1246,7 @@ checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -1184,13 +1287,24 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] +[[package]] +name = "pulldown-cmark" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" +dependencies = [ + "bitflags 2.3.3", + "memchr", + "unicase", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1199,9 +1313,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.31" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -1380,6 +1494,24 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scc" +version = "2.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d25269dd3a12467afe2e510f69fb0b46b698e5afb296b59f2145259deaf8e8" +dependencies = [ + "sdd", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1396,24 +1528,39 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sdd" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49c1eeaf4b6a87c7479688c6d52b9f1153cedd3c489300564f932b065c6eab95" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +dependencies = [ + "serde", +] + [[package]] name = "serde" -version = "1.0.171" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -1436,6 +1583,31 @@ dependencies = [ "serde", ] +[[package]] +name = "serial_test" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b4b487fe2acf240a021cf57c6b2b4903b1e78ca0ecd862a71b71d2a51fed77d" +dependencies = [ + "futures", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82fe9db325bcef1fbcde82e078a5cc4efdf787e96b3b9cf45b50b529f2083d67" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "sha-1" version = "0.10.1" @@ -1482,6 +1654,21 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +[[package]] +name = "skeptic" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" +dependencies = [ + "bytecount", + "cargo_metadata", + "error-chain", + "glob", + "pulldown-cmark", + "tempfile", + "walkdir", +] + [[package]] name = "slab" version = "0.4.8" @@ -1541,7 +1728,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -1585,15 +1772,34 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.26" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" version = "1.0.43" @@ -1611,7 +1817,7 @@ checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -1678,7 +1884,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -1783,7 +1989,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", ] [[package]] @@ -1838,6 +2044,12 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "triomphe" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" + [[package]] name = "trust-dns-proto" version = "0.22.0" @@ -1895,6 +2107,12 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +[[package]] +name = "unicase" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" + [[package]] name = "unicode-bidi" version = "0.3.13" @@ -1951,6 +2169,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -1993,7 +2221,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -2015,7 +2243,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2067,6 +2295,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index af15ab4f6..077374244 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "1.2.0" +version = "1.3.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -55,6 +55,10 @@ tracing-subscriber = { version = "0.3.17", features = [ "std", ] } lru = "0.12.0" +mini-moka = "0.10.3" [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" + +[dev-dependencies] +serial_test = "*" diff --git a/charts/pgcat/Chart.yaml b/charts/pgcat/Chart.yaml index 430c839bf..01294788d 100644 --- a/charts/pgcat/Chart.yaml +++ b/charts/pgcat/Chart.yaml @@ -4,5 +4,5 @@ description: A Helm chart for PgCat a PostgreSQL pooler and proxy (like PgBounce maintainers: - name: PostgresML email: team@postgresml.org -appVersion: "1.2.0" +appVersion: "1.3.0" version: 0.2.5 diff --git a/charts/pgcat/templates/secret.yaml b/charts/pgcat/templates/secret.yaml index 358f4e97b..547f993fa 100644 --- a/charts/pgcat/templates/secret.yaml +++ b/charts/pgcat/templates/secret.yaml @@ -51,6 +51,10 @@ stringData: query_parser_enabled = {{ default true $pool.query_parser_enabled }} query_parser_read_write_splitting = {{ default true $pool.query_parser_read_write_splitting }} primary_reads_enabled = {{ default true $pool.primary_reads_enabled }} + db_activity_based_routing = {{ default false $pool.db_activity_based_routing }} + db_activity_based_ms_init_delay = {{ default 100 $pool.db_activity_based_ms_init_delay }} + db_activity_ttl = {{ default 900 $pool.db_activity_ttl }} + table_mutation_cache_ttl = {{ default 50 $pool.table_mutation_cache_ttl }} sharding_function = {{ default "pg_bigint_hash" $pool.sharding_function | quote }} {{- range $index, $user := $pool.users }} diff --git a/charts/pgcat/values.yaml b/charts/pgcat/values.yaml index 20a4e27b8..3f933ca65 100644 --- a/charts/pgcat/values.yaml +++ b/charts/pgcat/values.yaml @@ -298,6 +298,22 @@ configuration: # ## @param configuration.poolsPostgres.query_parser_read_write_splitting # query_parser_read_write_splitting: true + # ## Db activity based routing. If enabled, we'll route queries to the primary if the table was recently mutated. + # ## @param configuration.poolsPostgres.db_activity_based_routing + # db_activity_based_routing: false + + # ## DB activity based init delay. How long to wait before starting to route queries to the primary after a table mutation. + # ## @param configuration.poolsPostgres.db_activity_based_ms_init_delay + # db_activity_based_ms_init_delay: 100 + + # ## DB activity TTL. How long before marking the DB as inactive after no mutations or queries. + # ## @param configuration.poolsPostgres.db_activity_ttl + # db_activity_ttl: 900 + + # ## Table mutation cache TTL. How long to keep track of table mutations. + # ## @param configuration.poolsPostgres.table_mutation_cache_ttl + # table_mutation_cache_ttl: 50 + # ## If the query parser is enabled and this setting is enabled, the primary will be part of the pool of databases used for # ## load balancing of read queries. Otherwise, the primary will only be used for write # ## queries. The primary can always be explicitly selected with our custom protocol. diff --git a/src/config.rs b/src/config.rs index b0d98fb52..39eb11532 100644 --- a/src/config.rs +++ b/src/config.rs @@ -589,6 +589,19 @@ pub struct Pool { #[serde(default = "Pool::default_prepared_statements_cache_size")] pub prepared_statements_cache_size: usize, + // Support for query routing based on database activity + #[serde(default = "Pool::default_db_activity_based_routing")] + pub db_activity_based_routing: bool, + + #[serde(default = "Pool::default_db_activity_init_delay")] + pub db_activity_init_delay: u64, + + #[serde(default = "Pool::default_db_activity_ttl")] + pub db_activity_ttl: u64, + + #[serde(default = "Pool::default_table_mutation_cache_ms_ttl")] + pub table_mutation_cache_ms_ttl: u64, + pub plugins: Option, pub shards: BTreeMap, pub users: BTreeMap, @@ -642,6 +655,25 @@ impl Pool { 0 } + pub fn default_db_activity_based_routing() -> bool { + false + } + + pub fn default_db_activity_init_delay() -> u64 { + // 100 milliseconds + 100 + } + + pub fn default_db_activity_ttl() -> u64 { + // 15 minutes + 15 * 60 + } + + pub fn default_table_mutation_cache_ms_ttl() -> u64 { + // 50 milliseconds + 50 + } + pub fn validate(&mut self) -> Result<(), Error> { match self.default_role.as_ref() { "any" => (), @@ -724,6 +756,23 @@ impl Pool { user.validate()?; } + if self.db_activity_based_routing { + if self.db_activity_init_delay == 0 { + error!("db_activity_init_delay must be greater than 0"); + return Err(Error::BadConfig); + } + + if self.table_mutation_cache_ms_ttl == 0 { + error!("table_mutation_cache_ms_ttl must be greater than 0"); + return Err(Error::BadConfig); + } + + if self.db_activity_ttl == 0 { + error!("db_activity_ttl must be greater than 0"); + return Err(Error::BadConfig); + } + } + Ok(()) } } @@ -753,6 +802,10 @@ impl Default for Pool { cleanup_server_connections: true, log_client_parameter_status_changes: false, prepared_statements_cache_size: Self::default_prepared_statements_cache_size(), + db_activity_based_routing: Self::default_db_activity_based_routing(), + db_activity_init_delay: Self::default_db_activity_init_delay(), + db_activity_ttl: Self::default_db_activity_ttl(), + table_mutation_cache_ms_ttl: Self::default_table_mutation_cache_ms_ttl(), plugins: None, shards: BTreeMap::from([(String::from("1"), Shard::default())]), users: BTreeMap::default(), @@ -1289,6 +1342,22 @@ impl Config { "[pool: {}] Cleanup server connections: {}", pool_name, pool_config.cleanup_server_connections ); + info!( + "[pool: {}] DB activity based routing: {}", + pool_name, pool_config.db_activity_based_routing + ); + info!( + "[pool: {}] DB activity init delay: {}", + pool_name, pool_config.db_activity_init_delay + ); + info!( + "[pool: {}] DB activity TTL: {}", + pool_name, pool_config.db_activity_ttl + ); + info!( + "[pool: {}] Table mutation cache TTL: {}", + pool_name, pool_config.table_mutation_cache_ms_ttl + ); info!( "[pool: {}] Log client parameter status changes: {}", pool_name, pool_config.log_client_parameter_status_changes diff --git a/src/pool.rs b/src/pool.rs index 7915a0a41..e3ddc43bb 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -174,6 +174,18 @@ pub struct PoolSettings { // Read from the primary as well or not. pub primary_reads_enabled: bool, + // Automatic primary/replica selection based on recent activity. + pub db_activity_based_routing: bool, + + // DB activity init delay + pub db_activity_init_delay: u64, + + // DB activity TTL + pub db_activity_ttl: u64, + + // Table mutation cache TTL + pub table_mutation_cache_ms_ttl: u64, + // Sharding function. pub sharding_function: ShardingFunction, @@ -223,6 +235,10 @@ impl Default for PoolSettings { query_parser_max_length: None, query_parser_read_write_splitting: false, primary_reads_enabled: true, + db_activity_based_routing: false, + db_activity_init_delay: 100, + db_activity_ttl: 15 * 60, + table_mutation_cache_ms_ttl: 50, sharding_function: ShardingFunction::PgBigintHash, automatic_sharding_key: None, healthcheck_delay: General::default_healthcheck_delay(), @@ -537,6 +553,10 @@ impl ConnectionPool { .query_parser_read_write_splitting, primary_reads_enabled: pool_config.primary_reads_enabled, sharding_function: pool_config.sharding_function, + db_activity_based_routing: pool_config.db_activity_based_routing, + db_activity_init_delay: pool_config.db_activity_init_delay, + db_activity_ttl: pool_config.db_activity_ttl, + table_mutation_cache_ms_ttl: pool_config.table_mutation_cache_ms_ttl, automatic_sharding_key: pool_config.automatic_sharding_key.clone(), healthcheck_delay: config.general.healthcheck_delay, healthcheck_timeout: config.general.healthcheck_timeout, diff --git a/src/query_router.rs b/src/query_router.rs index 3e485a0dc..16683b719 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -2,6 +2,7 @@ /// or implied query characteristics. use bytes::{Buf, BytesMut}; use log::{debug, error}; +use mini_moka::sync::Cache; use once_cell::sync::OnceCell; use regex::{Regex, RegexSet}; use sqlparser::ast::Statement::{Delete, Insert, Query, StartTransaction, Update}; @@ -11,6 +12,7 @@ use sqlparser::ast::{ }; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; +use std::sync::OnceLock; use crate::config::Role; use crate::errors::Error; @@ -21,6 +23,7 @@ use crate::sharding::Sharder; use std::collections::BTreeSet; use std::io::Cursor; +use std::time::Duration; use std::{cmp, mem}; /// Regexes used to parse custom commands. @@ -66,6 +69,18 @@ static CUSTOM_SQL_REGEX_SET: OnceCell = OnceCell::new(); // Get the value inside the custom command. static CUSTOM_SQL_REGEX_LIST: OnceCell> = OnceCell::new(); +#[derive(Debug, Clone, PartialEq)] +enum DatabaseActivityState { + Active, + Initializing, +} + +// A moka cache for the databases +// the key is the database name and the value is the database activity state +static DATABASE_ACTIVITY_CACHE: OnceLock> = OnceLock::new(); +// A moka cache for the tables, the key is the db_table. +static TABLE_MUTATIONS_CACHE: OnceLock> = OnceLock::new(); + /// The query router. pub struct QueryRouter { /// Which shard we should be talking to right now. @@ -87,6 +102,12 @@ pub struct QueryRouter { placeholders: Vec, } +struct ExtractedExprsAndTables<'a> { + exprs: Vec, + table_names: Vec>, + assignments_opt: Option<&'a Vec>, +} + impl QueryRouter { /// One-time initialization of regexes /// that parse our custom SQL protocol. @@ -398,6 +419,41 @@ impl QueryRouter { } } + fn database_activity_cache(&self) -> Cache { + DATABASE_ACTIVITY_CACHE + .get_or_init(|| { + Cache::builder() + .time_to_idle(Duration::from_secs(self.pool_settings.db_activity_ttl)) + .build() + }) + .clone() + } + + /// Check database activity state and reset it if necessary + fn database_activity_state(&self, db: &String) -> DatabaseActivityState { + let cache = self.database_activity_cache(); + + // Exists in cache + if cache.contains_key(db) { + return cache.get(db).unwrap(); + } + + // Not in cache + debug!("Adding database to cache: {}", db); + + cache.insert(db.to_string(), DatabaseActivityState::Initializing); + + // Set a timer to update the cache + let db = db.clone(); + let db_activity_init_delay = self.pool_settings.db_activity_init_delay; + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(db_activity_init_delay)).await; + cache.insert(db, DatabaseActivityState::Active); + }); + + DatabaseActivityState::Initializing + } + /// Try to infer which server to connect to based on the contents of the query. pub fn infer(&mut self, ast: &Vec) -> Result<(), Error> { if !self.pool_settings.query_parser_read_write_splitting { @@ -412,9 +468,23 @@ impl QueryRouter { return Err(Error::QueryRouterParserError("empty query".into())); } + let mut primary_set_based_on_activity = false; let mut visited_write_statement = false; let mut prev_inferred_shard = None; + if self.pool_settings.db_activity_based_routing { + let db = self.pool_settings.db.clone(); + let state = self.database_activity_state(&db); + debug!("Database activity state: {:?}", state); + + if let DatabaseActivityState::Initializing = state { + debug!("Database is initializing, going to primary"); + + self.active_role = Some(Role::Primary); + primary_set_based_on_activity = true; + } + } + for q in ast { match q { // All transactions go to the primary, probably a write. @@ -425,6 +495,22 @@ impl QueryRouter { // Likely a read-only query Query(query) => { + if primary_set_based_on_activity { + // If we already set the role based on activity, we don't need to do it again + continue; + } + + if self.pool_settings.db_activity_based_routing { + // Check if the tables in the query have been written to recently + if self.query_handles_tables_in_mutation_cache(query) { + debug!("Query handles tables in mutation cache, going to primary"); + + self.active_role = Some(Role::Primary); + primary_set_based_on_activity = true; + continue; + } + } + match &self.pool_settings.automatic_sharding_key { Some(_) => { // TODO: if we have multiple queries in the same message, @@ -455,6 +541,13 @@ impl QueryRouter { // Likely a write _ => { + debug!("Write statement found, going to primary"); + + if self.pool_settings.db_activity_based_routing { + // add all of the query tables to the mutation cache + self.update_mutation_cache_on_write(q); + } + match &self.pool_settings.automatic_sharding_key { Some(_) => { // TODO: similar to the above, if we have multiple queries in the @@ -497,11 +590,40 @@ impl QueryRouter { Ok(()) } - fn infer_shard_on_write(&mut self, q: &Statement) -> Result, Error> { - let mut exprs = Vec::new(); + fn table_mutations_cache(&self) -> Cache { + TABLE_MUTATIONS_CACHE + .get_or_init(|| { + Cache::builder() + .time_to_live(Duration::from_millis( + self.pool_settings.table_mutation_cache_ms_ttl, + )) + .build() + }) + .clone() + } - // Collect all table names from the query. + fn query_handles_tables_in_mutation_cache(&self, query: &sqlparser::ast::Query) -> bool { + let table_mutations_cache = self.table_mutations_cache(); + debug!("Checking if query handles tables in mutation cache"); + debug!("Table mutations cache: {:?}", table_mutations_cache); + + for tables in self.table_names(query) { + for table in tables { + if table_mutations_cache.contains_key(&self.table_mutation_cache_key(table)) { + return true; + } + } + } + + false + } + fn extract_exprs_and_table_names<'a>( + &'a self, + q: &'a Statement, + ) -> Option> { + let mut exprs = Vec::new(); let mut table_names = Vec::new(); + let mut assignments_opt = None; match q { Insert(i) => { @@ -520,7 +642,7 @@ impl QueryRouter { exprs.push(expr.clone()); } - // Multi tables delete are not supported in postgres. + // Multi-tables delete are not supported in postgres. assert!(d.tables.is_empty()); if let Some(using_tbl_with_join) = &d.using { @@ -544,14 +666,55 @@ impl QueryRouter { Self::process_table_with_join(from_tbl, &mut exprs, &mut table_names); } Self::process_selection(selection, &mut exprs); + + assignments_opt = Some(assignments); + } + _ => return None, + }; + + Some(ExtractedExprsAndTables { + exprs, + table_names, + assignments_opt, + }) + } + + fn infer_shard_on_write(&mut self, q: &Statement) -> Result, Error> { + if let Some(extracted) = self.extract_exprs_and_table_names(q) { + let exprs = extracted.exprs; + let table_names = extracted.table_names; + let assignments_opt = extracted.assignments_opt; + + if let Some(assignments) = assignments_opt { self.assignment_parser(assignments)?; } - _ => { - return Ok(None); + + Ok(self.infer_shard_from_exprs(exprs, table_names)) + } else { + Ok(None) + } + } + + fn update_mutation_cache_on_write(&self, q: &Statement) { + if let Some(extracted) = self.extract_exprs_and_table_names(q) { + debug!("Updating mutation cache on write"); + + let table_names = extracted.table_names; + debug!("Table names in mutation query: {:?}", table_names); + let table_mutations_cache = self.table_mutations_cache(); + for tables in table_names { + for table in tables { + table_mutations_cache.insert(self.table_mutation_cache_key(table), true); + } } - }; + } + } - Ok(self.infer_shard_from_exprs(exprs, table_names)) + // combines the database name and table name into a single string + // to be used as the key in the table mutation cache + // e.g. "mydb.mytable" + fn table_mutation_cache_key(&self, table: Ident) -> String { + format!("{}.{}", self.pool_settings.db, table.value) } fn process_query( @@ -939,6 +1102,18 @@ impl QueryRouter { self.infer_shard_from_exprs(exprs, table_names) } + /// get table names from query + fn table_names(&self, query: &sqlparser::ast::Query) -> Vec> { + let mut exprs = Vec::new(); + + let mut table_names = Vec::new(); + Self::process_query(query, &mut exprs, &mut table_names, &None); + + debug!("Table names in query: {:?}", table_names); + + table_names + } + fn infer_shard_from_exprs( &mut self, exprs: Vec, @@ -1106,6 +1281,7 @@ mod test { use crate::messages::simple_query; use crate::sharding::ShardingFunction; use bytes::BufMut; + use serial_test::serial; #[test] fn test_defaults() { @@ -1461,6 +1637,10 @@ mod test { auth_query_password: None, auth_query_user: None, db: "test".to_string(), + db_activity_based_routing: PoolSettings::default().db_activity_based_routing, + db_activity_init_delay: PoolSettings::default().db_activity_init_delay, + db_activity_ttl: PoolSettings::default().db_activity_ttl, + table_mutation_cache_ms_ttl: PoolSettings::default().table_mutation_cache_ms_ttl, plugins: None, }; let mut qr = QueryRouter::new(); @@ -1539,6 +1719,10 @@ mod test { auth_query_password: None, auth_query_user: None, db: "test".to_string(), + db_activity_based_routing: PoolSettings::default().db_activity_based_routing, + db_activity_init_delay: PoolSettings::default().db_activity_init_delay, + db_activity_ttl: PoolSettings::default().db_activity_ttl, + table_mutation_cache_ms_ttl: PoolSettings::default().table_mutation_cache_ms_ttl, plugins: None, }; @@ -1954,4 +2138,150 @@ mod test { assert_eq!(res, Ok(PluginOutput::Allow)); } + + #[tokio::test] + #[serial] + async fn test_db_activity_based_routing_initializing_state() { + QueryRouter::setup(); + let mut qr = QueryRouter::new(); + qr.pool_settings.db_activity_based_routing = true; + qr.pool_settings.query_parser_read_write_splitting = true; + qr.pool_settings.query_parser_enabled = true; + qr.pool_settings.db = "test_table_mutation_cache".to_string(); + + qr.database_activity_cache() + .invalidate(&qr.pool_settings.db.clone()); + + let query = simple_query("SELECT * FROM some_table"); + let ast = qr.parse(&query).unwrap(); + + // Initially, the database activity should be in the "Initializing" state + let state = qr.database_activity_state(&qr.pool_settings.db.clone()); + assert_eq!(state, DatabaseActivityState::Initializing); + + // Check that the router chooses the primary role due to "Initializing" state + assert!(qr.infer(&ast).is_ok()); + assert_eq!(qr.role(), Some(Role::Primary)); + } + + #[tokio::test] + #[serial] + async fn test_db_activity_based_routing_active_state() { + QueryRouter::setup(); + let mut qr = QueryRouter::new(); + qr.pool_settings.db_activity_based_routing = true; + qr.pool_settings.query_parser_read_write_splitting = true; + qr.pool_settings.query_parser_enabled = true; + qr.pool_settings.db = "test_table_mutation_cache".to_string(); + + let db_name = qr.pool_settings.db.clone(); + let cache = qr.database_activity_cache(); + cache.insert(db_name.clone(), DatabaseActivityState::Active); + + let query = simple_query("SELECT * FROM some_table"); + let ast = qr.parse(&query).unwrap(); + + // Check that the router can choose a replica role when in "Active" state + assert!(qr.infer(&ast).is_ok()); + assert_eq!(qr.role(), None); // Default should allow replica due to active state + } + + #[tokio::test] + #[serial] + async fn test_table_mutation_cache_on_write() { + QueryRouter::setup(); + let mut qr = QueryRouter::new(); + qr.pool_settings.db_activity_based_routing = true; + qr.pool_settings.table_mutation_cache_ms_ttl = 20_000; // 20 seconds in milliseconds + qr.pool_settings.query_parser_enabled = true; + qr.pool_settings.query_parser_read_write_splitting = true; + qr.pool_settings.db = "test_table_mutation_cache".to_string(); + + qr.database_activity_cache() + .invalidate(&qr.pool_settings.db.clone()); + + let query = simple_query("UPDATE some_table SET col1 = 'value' WHERE col2 = 1"); + let ast = qr.parse(&query).unwrap(); + + // Simulate the mutation query which should populate the mutation cache + assert!(qr.infer(&ast).is_ok()); + assert_eq!(qr.role(), Some(Role::Primary)); + + let table_cache_key = qr.table_mutation_cache_key(Ident::new("some_table")); + let cache = qr.table_mutations_cache(); + + // Ensure the table mutation cache contains the table with recent write + assert!(cache.contains_key(&table_cache_key)); + } + + #[tokio::test] + #[serial] + async fn test_db_activity_based_routing_multi_query() { + use super::*; + use crate::messages::simple_query; + use tokio::time::Duration; + + QueryRouter::setup(); + let mut qr = QueryRouter::new(); + + // Configure the pool settings for db_activity_based_routing + qr.pool_settings.query_parser_read_write_splitting = true; + qr.pool_settings.query_parser_enabled = true; + qr.pool_settings.db_activity_based_routing = true; + qr.pool_settings.db = "test_db_activity_routing".to_string(); + + qr.database_activity_cache() + .invalidate(&qr.pool_settings.db.clone()); + + // First query when database is initializing + let query = simple_query("SELECT * FROM test_table"); + let ast = qr.parse(&query).unwrap(); + assert!(qr.infer(&ast).is_ok()); + // Should route to primary because database is initializing + assert_eq!(qr.role(), Some(Role::Primary)); + + // Wait for the initialization delay to pass + tokio::time::sleep(Duration::from_millis( + qr.pool_settings.db_activity_init_delay * 2, + )) + .await; + + // Next query after database is active + let query = simple_query("SELECT * FROM test_table"); + let ast = qr.parse(&query).unwrap(); + qr.active_role = None; // Reset the active_role + assert!(qr.infer(&ast).is_ok()); + // Should route to replica because database is active and no recent mutations + assert_eq!(qr.role(), None); + + // Simulate a write query to update the mutation cache + let query = simple_query("INSERT INTO test_table (id, name) VALUES (1, 'test')"); + let ast = qr.parse(&query).unwrap(); + qr.active_role = None; // Reset the active_role + assert!(qr.infer(&ast).is_ok()); + // Should route to primary because it's a write operation + assert_eq!(qr.role(), Some(Role::Primary)); + + // Immediately run a read query on the same table + let query = simple_query("SELECT * FROM test_table WHERE id = 1"); + let ast = qr.parse(&query).unwrap(); + qr.active_role = None; // Reset the active_role + assert!(qr.infer(&ast).is_ok()); + // Should route to primary because the table was recently mutated + assert_eq!(qr.role(), Some(Role::Primary)); + + // Wait for the mutation cache TTL to expire + tokio::time::sleep(Duration::from_millis( + qr.pool_settings.table_mutation_cache_ms_ttl * 2, + )) + .await; + + // Run the read query again after cache expiration + let query = simple_query("SELECT * FROM test_table WHERE id = 1"); + let ast = qr.parse(&query).unwrap(); + qr.active_role = None; // Reset the active_role + assert!(qr.infer(&ast).is_ok()); + // Should route to replica because mutation cache has expired + assert_eq!(qr.role(), None); + } } From 836ba94f8a7f9839e6a8e86a4d3870ede9b0ae05 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Thu, 9 Jan 2025 14:59:42 -0500 Subject: [PATCH 544/551] s/Iniitalize/Initialize/ (#897) --- src/logger.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logger.rs b/src/logger.rs index 867b1a24b..e206fc1fa 100644 --- a/src/logger.rs +++ b/src/logger.rs @@ -3,7 +3,7 @@ use tracing_subscriber; use tracing_subscriber::EnvFilter; pub fn init(args: &Args) { - // Iniitalize a default filter, and then override the builtin default "warning" with our + // Initialize a default filter, and then override the builtin default "warning" with our // commandline, (default: "info") let filter = EnvFilter::from_default_env().add_directive(args.log_level.into()); From aba64d6f39f8ae7594f0f392eaf251fc46223b25 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Thu, 27 Feb 2025 13:17:00 -0600 Subject: [PATCH 545/551] Add checkout_failure_limit config/feature (#911) In a high availability deployment of PgCat, it is possible that a client may land on a container of PgCat that is very busy with clients and as such the new client might be perpetually stuck in checkout failure loop because all connections are used by other clients. This is specially true in session mode pools with long-lived client connections (e.g. FDW connections). One way to fix this issue is to close client connections after they encounter some number of checkout failure. This will force the client to hit the Network load balancer again, land on a different process/container, try to checkout a connection on the new process/container. if it fails, it is disconnected and tries with another one. This mechanism is guaranteed to eventually land on a balanced state where all clients are able to find connections provided that the overall number of connections across all containers matches the number of clients. I was able to reproduce this issue in a control environment and was able to show this PR is able to fix it. --- CONFIG.md | 13 ++++++ src/client.rs | 22 +++++++++- src/config.rs | 20 +++++++++ src/pool.rs | 10 +++++ src/query_router.rs | 2 + tests/ruby/misc_spec.rb | 96 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 162 insertions(+), 1 deletion(-) diff --git a/CONFIG.md b/CONFIG.md index b6e16a7c8..a40dec329 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -298,6 +298,19 @@ Load balancing mode `random` selects the server at random `loc` selects the server with the least outstanding busy connections +### checkout_failure_limit +``` +path: pools..checkout_failure_limit +default: 0 (disabled) +``` + +`Maximum number of checkout failures a client is allowed before it +gets disconnected. This is needed to prevent persistent client/server +imbalance in high availability setups where multiple PgCat instances are placed +behind a single load balancer. If for any reason a client lands on a PgCat instance that has +a large number of connected clients, it might get stuck in perpetual checkout failure loop especially +in session mode +` ### default_role ``` path: pools..default_role diff --git a/src/client.rs b/src/client.rs index c226436e6..c72e9d2a2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -859,6 +859,8 @@ where // e.g. primary, replica, which shard. let mut query_router = QueryRouter::new(); + let mut checkout_failure_count: u64 = 0; + self.stats.register(self.stats.clone()); // Result returned by one of the plugins. @@ -1108,7 +1110,25 @@ where query_router.role(), err ); - + checkout_failure_count += 1; + if let Some(limit) = pool.settings.checkout_failure_limit { + if checkout_failure_count >= limit { + error!( + "Checkout failure limit reached ({} / {}) - disconnecting client", + checkout_failure_count, limit + ); + error_response_terminal( + &mut self.write, + &format!( + "checkout failure limit reached ({} / {})", + checkout_failure_count, limit + ), + ) + .await?; + self.stats.disconnect(); + return Ok(()); + } + } continue; } }; diff --git a/src/config.rs b/src/config.rs index 39eb11532..e56f92b9a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -558,6 +558,14 @@ pub struct Pool { /// Close idle connections that have been opened for longer than this. pub idle_timeout: Option, + /// Maximum number of checkout failures a client is allowed before it + /// gets disconnected. This is needed to prevent persistent client/server + /// imbalance in high availability setups where multiple PgCat instances are placed + /// behind a single load balancer. If for any reason a client lands on a PgCat instance that has + /// a large number of connected clients, it might get stuck in perpetual checkout failure loop especially + /// in session mode + pub checkout_failure_limit: Option, + /// Close server connections that have been opened for longer than this. /// Only applied to idle connections. If the connection is actively used for /// longer than this period, the pool will not interrupt it. @@ -782,6 +790,7 @@ impl Default for Pool { Pool { pool_mode: Self::default_pool_mode(), load_balancing_mode: Self::default_load_balancing_mode(), + checkout_failure_limit: None, default_role: String::from("any"), query_parser_enabled: false, query_parser_max_length: None, @@ -1298,6 +1307,17 @@ impl Config { None => self.general.idle_timeout, }; info!("[pool: {}] Idle timeout: {}ms", pool_name, idle_timeout); + match pool_config.checkout_failure_limit { + Some(checkout_failure_limit) => { + info!( + "[pool: {}] Checkout failure limit: {}", + pool_name, checkout_failure_limit + ); + } + None => { + info!("[pool: {}] Checkout failure limit: not set", pool_name); + } + }; info!( "[pool: {}] Sharding function: {}", pool_name, diff --git a/src/pool.rs b/src/pool.rs index e3ddc43bb..7ecf24c17 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -152,6 +152,14 @@ pub struct PoolSettings { /// Random or LeastOutstandingConnections. pub load_balancing_mode: LoadBalancingMode, + /// Maximum number of checkout failures a client is allowed before it + /// gets disconnected. This is needed to prevent persistent client/server + /// imbalance in high availability setups where multiple PgCat instances are placed + /// behind a single load balancer. If for any reason a client lands on a PgCat instance that has + /// a large number of connected clients, it might get stuck in perpetual checkout failure loop especially + /// in session mode + pub checkout_failure_limit: Option, + // Number of shards. pub shards: usize, @@ -227,6 +235,7 @@ impl Default for PoolSettings { PoolSettings { pool_mode: PoolMode::Transaction, load_balancing_mode: LoadBalancingMode::Random, + checkout_failure_limit: None, shards: 1, user: User::default(), db: String::default(), @@ -537,6 +546,7 @@ impl ConnectionPool { None => pool_config.pool_mode, }, load_balancing_mode: pool_config.load_balancing_mode, + checkout_failure_limit: pool_config.checkout_failure_limit, // shards: pool_config.shards.clone(), shards: shard_ids.len(), user: user.clone(), diff --git a/src/query_router.rs b/src/query_router.rs index 16683b719..a996e6611 100644 --- a/src/query_router.rs +++ b/src/query_router.rs @@ -1617,6 +1617,7 @@ mod test { let pool_settings = PoolSettings { pool_mode: PoolMode::Transaction, load_balancing_mode: crate::config::LoadBalancingMode::Random, + checkout_failure_limit: None, shards: 2, user: crate::config::User::default(), default_role: Some(Role::Replica), @@ -1699,6 +1700,7 @@ mod test { let pool_settings = PoolSettings { pool_mode: PoolMode::Transaction, load_balancing_mode: crate::config::LoadBalancingMode::Random, + checkout_failure_limit: Some(10), shards: 5, user: crate::config::User::default(), default_role: Some(Role::Replica), diff --git a/tests/ruby/misc_spec.rb b/tests/ruby/misc_spec.rb index aa17e8ec7..7cb0dc0ea 100644 --- a/tests/ruby/misc_spec.rb +++ b/tests/ruby/misc_spec.rb @@ -188,6 +188,102 @@ end end + describe "Checkout failure limit" do + context "when no checkout failure limit is set" do + before do + new_configs = processes.pgcat.current_config + new_configs["general"]["connect_timeout"] = 200 + new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + sleep 0.5 + end + + it "does not disconnect client" do + Array.new(5) do + Thread.new do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + for i in 0..4 + begin + conn.async_exec("SELECT pg_sleep(0.5);") + expect(conn.status).to eq(PG::CONNECTION_OK) + rescue PG::SystemError + expect(conn.status).to eq(PG::CONNECTION_OK) + end + end + conn.close + end + end.each(&:join) + end + end + + context "when checkout failure limit is set high" do + before do + new_configs = processes.pgcat.current_config + new_configs["general"]["connect_timeout"] = 200 + new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 + new_configs["pools"]["sharded_db"]["checkout_failure_limit"] = 10000 + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + sleep 0.5 + end + + it "does not disconnect client" do + Array.new(5) do + Thread.new do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + for i in 0..4 + begin + conn.async_exec("SELECT pg_sleep(0.5);") + expect(conn.status).to eq(PG::CONNECTION_OK) + rescue PG::SystemError + expect(conn.status).to eq(PG::CONNECTION_OK) + end + end + conn.close + end + end.each(&:join) + end + end + + context "when checkout failure limit is set low" do + before do + new_configs = processes.pgcat.current_config + new_configs["general"]["connect_timeout"] = 200 + new_configs["pools"]["sharded_db"]["users"]["0"]["pool_size"] = 1 + new_configs["pools"]["sharded_db"]["checkout_failure_limit"] = 2 + processes.pgcat.update_config(new_configs) + processes.pgcat.reload_config + sleep 0.5 + end + + it "disconnects client after reaching limit" do + Array.new(5) do + Thread.new do + conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) + checkout_failure_count = 0 + for i in 0..4 + begin + conn.async_exec("SELECT pg_sleep(1);") + expect(conn.status).to eq(PG::CONNECTION_OK) + rescue PG::SystemError + checkout_failure_count += 1 + expect(conn.status).to eq(PG::CONNECTION_OK) + rescue PG::ConnectionBad + expect(checkout_failure_count).to eq(2) + expect(conn.status).to eq(PG::CONNECTION_BAD) + break + end + end + conn.close + end + end.each(&:join) + puts processes.pgcat.logs + + end + end + end + describe "Server version reporting" do it "reports correct version for normal and admin databases" do server_conn = PG::connect(processes.pgcat.connection_string("sharded_db", "sharding_user")) From 4e4e36ca22978e41ca8d23bd07f4492b749f81cd Mon Sep 17 00:00:00 2001 From: "(Alan) Jia Lin Yuan" <33004730+JaLnYn@users.noreply.github.com> Date: Thu, 27 Feb 2025 15:30:45 -0500 Subject: [PATCH 546/551] Bump Rust Version on Dockerfile from 1.79.0 to 1.81 (#912) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8a7edeae9..536efe695 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.79.0-slim-bookworm AS builder +FROM rust:1.81.0-slim-bookworm AS builder RUN apt-get update && \ apt-get install -y build-essential From 0a9f1fccdb86e64c039815f48e37d1263dc55a2f Mon Sep 17 00:00:00 2001 From: Mostafa Date: Thu, 27 Feb 2025 14:49:25 -0600 Subject: [PATCH 547/551] Bump rust version on CI image (#913) --- Dockerfile.ci | 2 +- tests/docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 565418d41..b8a7180f7 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM cimg/rust:1.79.0 +FROM cimg/rust:1.81.0 COPY --from=sclevine/yj /bin/yj /bin/yj RUN /bin/yj -h RUN sudo apt-get update && \ diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 6eccddd17..ec77a2e11 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:bullseye +FROM rust:1.81.0-slim-bookworm COPY --from=sclevine/yj /bin/yj /bin/yj RUN /bin/yj -h From 853581f3ad030d335678021bb52da801003c45ec Mon Sep 17 00:00:00 2001 From: Nathanael Aninweze Date: Wed, 15 Apr 2026 19:41:11 -0400 Subject: [PATCH 548/551] Handle Flush ('H') extended-protocol message postgres.js (and likely other drivers) sends Flush between Parse/Describe and Bind/Execute when describeFirst is enabled (the default for parameterized non-prepared queries). pgcat was logging 'Unexpected code: H' and dropping the message, leaving the client hung waiting for the ParameterDescription/RowDescription response that never arrived. Add a Flush arm that drains the buffered extended-protocol messages into self.buffer (same as Sync), appends the Flush byte, sends to the server, forwards the server response, and keeps the server checked out so the subsequent Bind/Execute/Sync can complete on the same backend. --- src/client.rs | 125 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/src/client.rs b/src/client.rs index c72e9d2a2..c8539a3ed 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1347,6 +1347,131 @@ where .push_back(ExtendedProtocolData::create_new_close(message, close)); } + // Flush + // Frontend asks the server to push pending responses + // without ending the extended-query sequence. The + // server stays in extended-query mode (no + // ReadyForQuery) and we keep the server checked out. + // Used by drivers like postgres.js between + // Parse/Describe and Bind/Execute (`describeFirst`). + 'H' => { + debug!("Flushing buffered extended-protocol messages to server"); + + // Drain the buffered extended-protocol messages + // into self.buffer the same way Sync does, but + // without sending Sync — append the Flush byte + // instead so the server pushes responses but + // stays in extended-query state. + while let Some(protocol_data) = + self.extended_protocol_data_buffer.pop_front() + { + match protocol_data { + ExtendedProtocolData::Parse { data, metadata } => { + let (parse, hash) = match metadata { + Some(metadata) => metadata, + None => { + let first_char_in_name = *data.get(5).unwrap_or(&0); + if first_char_in_name != 0 { + server.mark_dirty(); + } + self.buffer.put(&data[..]); + continue; + } + }; + + if server.has_prepared_statement(&parse.name) { + self.response_message_queue_buffer.put(parse_complete()); + } else { + self.register_parse_to_server_cache( + false, &hash, &parse, &pool, server, &address, + ) + .await?; + self.buffer.put(&data[..]); + } + } + ExtendedProtocolData::Bind { data, metadata } => { + if let Some(client_given_name) = metadata { + self.ensure_prepared_statement_is_on_server( + client_given_name, + &pool, + server, + &address, + ) + .await?; + } + self.buffer.put(&data[..]); + } + ExtendedProtocolData::Describe { data, metadata } => { + if let Some(client_given_name) = metadata { + self.ensure_prepared_statement_is_on_server( + client_given_name, + &pool, + server, + &address, + ) + .await?; + } + self.buffer.put(&data[..]); + } + ExtendedProtocolData::Execute { data } => { + self.buffer.put(&data[..]) + } + ExtendedProtocolData::Close { data, close } => { + if self.prepared_statements_enabled + && close.is_prepared_statement() + && !close.anonymous() + { + self.prepared_statements.remove(&close.name); + self.response_message_queue_buffer.put(close_complete()); + } else { + self.buffer.put(&data[..]); + } + } + } + } + + // Append the Flush byte so the server pushes its + // pending responses now. + self.buffer.put(&message[..]); + + // If the buffer contains only the Flush byte (no + // pending extended-protocol work), there is + // nothing for the server to flush — just emit any + // queued client responses and continue. + let only_flush = *self.buffer.first().unwrap() == b'H'; + + if !self.response_message_queue_buffer.is_empty() { + if let Err(err) = write_all_flush( + &mut self.write, + &self.response_message_queue_buffer, + ) + .await + { + server.mark_bad(err.to_string().as_str()); + return Err(err); + } + self.response_message_queue_buffer.clear(); + } + + if !only_flush { + self.send_and_receive_loop( + code, + None, + server, + &address, + &pool, + &self.stats.clone(), + ) + .await?; + } + + self.buffer.clear(); + + // Do NOT release the server: extended-query + // exchange is still open. A Sync from the client + // will eventually end it. + } + // Sync // Frontend (client) is asking for the query result now. 'S' => { From 6b5fd4c54f091bc8e8bcbf8f7b0dd1d742fbe105 Mon Sep 17 00:00:00 2001 From: Nathanael Aninweze Date: Wed, 15 Apr 2026 20:41:15 -0400 Subject: [PATCH 549/551] Read Flush responses without waiting for ReadyForQuery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit added a Flush arm in the client loop but reused send_and_receive_loop, which calls Server::recv() — that recv loop only terminates on ReadyForQuery ('Z'). Servers do NOT send ReadyForQuery in response to Flush, so the read hung forever. Add Server::recv_flush_response() that breaks on RowDescription, NoData, ErrorResponse, or BindComplete (the terminal describe / bind response markers). Wire it into the Flush arm via a direct send + recv_flush_response call path. Verified locally: postgres.js with prepare:false + params now completes in ~500ms through pgcat (previously hung). --- src/client.rs | 36 +++++++++++++++------ src/server.rs | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 9 deletions(-) diff --git a/src/client.rs b/src/client.rs index c8539a3ed..971e7d494 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1454,15 +1454,33 @@ where } if !only_flush { - self.send_and_receive_loop( - code, - None, - server, - &address, - &pool, - &self.stats.clone(), - ) - .await?; + // Send to server. + self.send_server_message(server, &self.buffer, &address, &pool) + .await?; + + // Read the Flush response — must NOT use the + // normal recv loop because the server does not + // send ReadyForQuery in response to Flush. + let response = match server + .recv_flush_response(Some(&mut self.server_parameters)) + .await + { + Ok(r) => r, + Err(err) => { + pool.ban(&address, BanReason::MessageReceiveFailed, Some(&self.stats)); + error_response_terminal( + &mut self.write, + &format!("error receiving Flush response: {:?}", err), + ) + .await?; + return Err(err); + } + }; + + if let Err(err) = write_all_flush(&mut self.write, &response).await { + server.mark_bad(err.to_string().as_str()); + return Err(err); + } } self.buffer.clear(); diff --git a/src/server.rs b/src/server.rs index 882450eaf..54eaa48db 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1114,6 +1114,92 @@ impl Server { Ok(bytes) } + /// Read server messages emitted in response to a Flush. + /// + /// Unlike `recv`, this loop never waits for ReadyForQuery (`Z`) — + /// the server does not send one in response to Flush. Instead we + /// read every message available, and stop once we hit a terminal + /// describe-response marker (RowDescription `T`, NoData `n`, + /// ParameterDescription `t`-followed-by-T-or-n, or ErrorResponse + /// `E`). On error we still keep buffering until ReadyForQuery + /// is NOT expected — we just return what we have. + pub async fn recv_flush_response( + &mut self, + mut client_server_parameters: Option<&mut ServerParameters>, + ) -> Result { + let mut saw_terminal = false; + loop { + let mut message = match read_message(&mut self.stream).await { + Ok(message) => message, + Err(err) => { + error!( + "Terminating server {:?} during Flush response: {:?}", + self.address, err + ); + self.bad = true; + return Err(err); + } + }; + + self.buffer.put(&message[..]); + let code = message.get_u8() as char; + let _len = message.get_i32(); + + match code { + // ParameterStatus — track session params. + 'S' => { + let key = message.read_string().unwrap(); + let value = message.read_string().unwrap(); + if let Some(client_server_parameters) = client_server_parameters.as_mut() { + client_server_parameters.set_param(key.clone(), value.clone(), false); + } + self.server_parameters.set_param(key, value, false); + } + + // ParseComplete — describe-only flush returns this alone. + '1' => { + self.registering_prepared_statement.pop_front(); + // If client only sent Parse+Flush, no further messages + // are coming. But normally client also sent Describe, + // so keep reading for ParameterDescription/RowDescription. + } + + // Terminal describe-response markers. + // RowDescription / NoData / ErrorResponse — once the + // server has emitted one of these the describe round-trip + // is complete. + 'T' | 'n' | 'E' => { + saw_terminal = true; + self.data_available = false; + break; + } + + // ParameterDescription — sent before RowDescription/NoData. + // Keep reading. + 't' => (), + + // BindComplete — possible if Flush follows Bind. + '2' => { + saw_terminal = true; + self.data_available = false; + break; + } + + _ => (), + } + + if saw_terminal { + break; + } + } + + let bytes = self.buffer.clone(); + self.stats().data_received(bytes.len()); + self.buffer.clear(); + self.last_activity = SystemTime::now(); + Ok(bytes) + } + // Determines if the server already has a prepared statement with the given name // Increments the prepared statement cache hit counter pub fn has_prepared_statement(&mut self, name: &str) -> bool { From cb3a205f3fb263e4614227cc22da281dcae5ff6d Mon Sep 17 00:00:00 2001 From: nathanaelaninweze Date: Wed, 20 May 2026 23:37:31 -0400 Subject: [PATCH 550/551] Flush response: drop double-copy parse and clone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - recv_flush_response: skip body parse for non-S/non-1 codes (5–N byte protocol frames; framing byte is enough to dispatch). - recv_flush_response: replace self.buffer.clone() with std::mem::take to avoid one memcpy+alloc per Flush response. - Merge separate saw_terminal flag into single matches!() check on the framing byte. Net effect on cloudrest describe-heavy workload: removes one full-buffer clone and one redundant get_u8/get_i32/read_string pass per Flush. At intra-DC RTT (~0.3ms) and 10k+ qps the saved CPU is the dominant win. --- src/server.rs | 79 ++++++++++++++++++++++----------------------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/src/server.rs b/src/server.rs index 54eaa48db..42992cae5 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1127,9 +1127,8 @@ impl Server { &mut self, mut client_server_parameters: Option<&mut ServerParameters>, ) -> Result { - let mut saw_terminal = false; loop { - let mut message = match read_message(&mut self.stream).await { + let message = match read_message(&mut self.stream).await { Ok(message) => message, Err(err) => { error!( @@ -1141,61 +1140,51 @@ impl Server { } }; - self.buffer.put(&message[..]); - let code = message.get_u8() as char; - let _len = message.get_i32(); - - match code { - // ParameterStatus — track session params. - 'S' => { - let key = message.read_string().unwrap(); - let value = message.read_string().unwrap(); + // Inspect framing byte without consuming the message — avoid + // re-parsing length and avoid the double copy of body bytes. + let code = *message.first().unwrap_or(&0); + + // Only ParameterStatus needs body parsing (to track session + // params). Skip get_u8/get_i32/read_string for everything + // else — these messages are 5–N bytes and we only need to + // know the framing byte. + if code == b'S' { + // ParameterStatus body: code(1) + len(4) + key\0 + value\0. + // Parse into an owned BytesMut just for the body so we + // can use the existing BytesMutReader impl without + // disturbing the original `message` we still need to + // append to self.buffer below. + let mut body = BytesMut::from(&message[5..]); + if let (Ok(key), Ok(value)) = (body.read_string(), body.read_string()) { if let Some(client_server_parameters) = client_server_parameters.as_mut() { client_server_parameters.set_param(key.clone(), value.clone(), false); } self.server_parameters.set_param(key, value, false); } - - // ParseComplete — describe-only flush returns this alone. - '1' => { - self.registering_prepared_statement.pop_front(); - // If client only sent Parse+Flush, no further messages - // are coming. But normally client also sent Describe, - // so keep reading for ParameterDescription/RowDescription. - } - - // Terminal describe-response markers. - // RowDescription / NoData / ErrorResponse — once the - // server has emitted one of these the describe round-trip - // is complete. - 'T' | 'n' | 'E' => { - saw_terminal = true; - self.data_available = false; - break; - } - - // ParameterDescription — sent before RowDescription/NoData. - // Keep reading. - 't' => (), - - // BindComplete — possible if Flush follows Bind. - '2' => { - saw_terminal = true; - self.data_available = false; - break; - } - - _ => (), + } else if code == b'1' { + // ParseComplete — consume one pending registration. + self.registering_prepared_statement.pop_front(); } - if saw_terminal { + self.buffer.extend_from_slice(&message); + + // Terminal describe/bind-response markers — server done + // emitting in response to this Flush. + // T = RowDescription, n = NoData, E = ErrorResponse, + // 2 = BindComplete (Flush after Bind w/o Execute). + if matches!(code, b'T' | b'n' | b'E' | b'2') { + self.data_available = false; break; } } - let bytes = self.buffer.clone(); + // mem::take instead of clone — hands the BytesMut to caller + // without a memcpy+alloc. self.buffer is left empty with retained + // capacity is NOT a concern; the empty BytesMut here is fine + // because we reuse it next call (push triggers fresh alloc on + // first put, identical cost to the prior clear()). + let bytes = std::mem::take(&mut self.buffer); self.stats().data_received(bytes.len()); - self.buffer.clear(); self.last_activity = SystemTime::now(); Ok(bytes) } From 1ff5d18b0674581c7826ad2aef4c0551bb629a76 Mon Sep 17 00:00:00 2001 From: Nathanael Aninweze Date: Tue, 26 May 2026 18:50:20 -0400 Subject: [PATCH 551/551] client: don't silently drop client-side prepared statement on server re-prepare failure When a checked-out backend rejected re-Parse of a previously-prepared statement (e.g. after schema-change cleanup, DEALLOCATE, or transient backend state), the old code removed the entry from the *client* prepared_statements HashMap and returned Ok(). This left tokio-postgres clients in an unrecoverable state: the client still believed the statement was valid (it never received a Close), so its very next Bind for that name missed in buffer_bind and the whole TCP conn dropped with 'Prepared statement sN doesn't exist'. For long-lived ETL clients (pgraft's tokio-postgres prepares many shapes via execute() and caches them per-conn by SQL), this killed every write inflight on that conn and orphaned any open COPY-IN, leaving AccessExclusiveLock on temp staging tables until session termination. New behavior: keep the client cache intact (it mirrors the client's own view), mark the misbehaving server bad so the pool replaces it, and propagate PreparedStatementError so the caller can fail this one operation and retry on a fresh backend. --- src/client.rs | 48 ++++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/client.rs b/src/client.rs index 971e7d494..8bb523305 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1886,29 +1886,37 @@ where .register_parse_to_server_cache(true, hash, parse, pool, server, address) .await { - Ok(_) => (), - Err(err) => match err { - Error::PreparedStatementError => { - debug!("Removed {} from client cache", client_name); - self.prepared_statements.remove(&client_name); - } - - _ => { - return Err(err); - } - }, + Ok(_) => Ok(()), + Err(Error::PreparedStatementError) => { + // The backend rejected our Parse. This used to silently + // drop the statement from the *client* cache and return + // Ok, which left tokio-postgres clients believing the + // statement was still valid: the very next Bind would + // miss in `buffer_bind` and abort the whole TCP conn + // with "Prepared statement sN doesn't exist", killing + // pgraft-style ETL workloads that prepare-then-reuse + // across many transactions. + // + // Keep the client cache intact (it represents what the + // client believes), mark this server bad so the pool + // replaces it on next checkout, and surface the error + // so the caller can retry on a fresh backend. + warn!( + "Server {:?} rejected re-prepare of `{}` — marking bad so pool replaces it", + address, client_name + ); + server.mark_bad("prepared statement re-register failed"); + Err(Error::PreparedStatementError) + } + Err(err) => Err(err), } } - None => { - return Err(Error::ClientError(format!( - "prepared statement `{}` not found", - client_name - ))) - } - }; - - Ok(()) + None => Err(Error::ClientError(format!( + "prepared statement `{}` not found", + client_name + ))), + } } /// Register the parse to the server cache and send it to the server if requested (ie. requested by pgcat)