diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5e3efdcb2ab..79b2683a453 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -97,7 +97,7 @@ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS pg18 -RUN MAKEFLAGS="-j $(nproc)" pgenv build 18.1 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 18.3 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install @@ -216,7 +216,7 @@ COPY --chown=citus:citus .psqlrc . RUN sudo chown --from=root:root citus:citus -R ~ # sets default pg version -RUN pgenv switch 18.1 +RUN pgenv switch 18.3 # make connecting to the coordinator easy ENV PGPORT=9700 diff --git a/.devcontainer/src/test/regress/Pipfile b/.devcontainer/src/test/regress/Pipfile index 8ade5491338..06e820beb44 100644 --- a/.devcontainer/src/test/regress/Pipfile +++ b/.devcontainer/src/test/regress/Pipfile @@ -16,7 +16,7 @@ tornado = ">=6.5.1,<6.6.0" zstandard = ">=0.25.0" construct = "*" docopt = "==0.6.2" -cryptography = "==44.0.3" +cryptography = "==46.0.5" pytest = "*" psycopg = "*" filelock = "*" @@ -25,7 +25,7 @@ pytest-timeout = "*" pytest-xdist = "*" pytest-repeat = "*" pyyaml = "*" -werkzeug = "==3.1.4" +werkzeug = "==3.1.5" "typing-extensions" = ">=4.13.2,<5" pyperclip = "==1.9.0" diff --git a/.devcontainer/src/test/regress/Pipfile.lock b/.devcontainer/src/test/regress/Pipfile.lock index 5b662d4343f..8d7fadb409d 100644 --- a/.devcontainer/src/test/regress/Pipfile.lock +++ b/.devcontainer/src/test/regress/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5f734fff88a49010712613a14addae5f38347c691d0b3004b09f59ba5c7cc061" + "sha256": "92e863acf1c77d535dc7da881b4e4cf03bb8fa748769c845aaee27d86d94657c" }, "pipfile-spec": 6, "requires": { @@ -292,11 +292,11 @@ }, "certifi": { "hashes": [ - "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", - "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120" + "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", + "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" ], "markers": "python_version >= '3.7'", - "version": "==2026.1.4" + "version": "==2026.2.25" }, "cffi": { "hashes": [ @@ -407,47 +407,59 @@ }, "cryptography": { "hashes": [ - "sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259", - "sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43", - "sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645", - "sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8", - "sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44", - "sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d", - "sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f", - "sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d", - "sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54", - "sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9", - "sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137", - "sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f", - "sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c", - "sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334", - "sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c", - "sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b", - "sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2", - "sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375", - "sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88", - "sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5", - "sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647", - "sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c", - "sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359", - "sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5", - "sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d", - "sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028", - "sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01", - "sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904", - "sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d", - "sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93", - "sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06", - "sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff", - "sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76", - "sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff", - "sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759", - "sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4", - "sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053" + "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", + "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", + "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9", + "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356", + "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257", + "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", + "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", + "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c", + "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", + "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", + "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", + "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", + "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", + "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", + "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", + "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4", + "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4", + "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", + "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", + "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", + "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678", + "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", + "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", + "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0", + "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", + "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d", + "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", + "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", + "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981", + "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", + "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", + "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2", + "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", + "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663", + "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", + "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", + "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", + "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a", + "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", + "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", + "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a", + "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826", + "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee", + "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", + "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648", + "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da", + "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", + "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", + "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87" ], "index": "pypi", - "markers": "python_version >= '3.7' and python_full_version not in '3.9.0, 3.9.1'", - "version": "==44.0.3" + "markers": "python_version >= '3.8' and python_full_version not in '3.9.0, 3.9.1'", + "version": "==46.0.5" }, "docopt": { "hashes": [ @@ -466,12 +478,12 @@ }, "filelock": { "hashes": [ - "sha256:a2241ff4ddde2a7cebddf78e39832509cb045d18ec1a09d7248d6bfc6bfbbe64", - "sha256:fbba7237d6ea277175a32c54bb71ef814a8546d8601269e1bfc388de333974e8" + "sha256:5ccf8069f7948f494968fc0713c10e5c182a9c9d9eef3a636307a20c2490f047", + "sha256:8f00faf3abf9dc730a1ffe9c354ae5c04e079ab7d3a683b7c32da5dd05f26af3" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==3.20.2" + "version": "==3.25.0" }, "flask": { "hashes": [ @@ -664,24 +676,24 @@ }, "mitmproxy-linux": { "hashes": [ - "sha256:0bea9353c71ebfd2174f6730b3fd0fdff3adea1aa15450035bed3b83e36ef455", - "sha256:2238455e65970382825baed2e998601ea82d8dcaae51bd8ee0859d596524a822", - "sha256:fbcb25316e95d0b2b5ced4e0cc3d90fdb1b7169300a005cc79339894d665363a" + "sha256:94b10fee02aa42287739623cef921e1a53955005d45c9e2fa309ae9f0bf8d37d", + "sha256:b4413e27c692f30036ad6d73432826e728ede026fac8e51651d0c545dd0177f2", + "sha256:ee842865a05f69196004ddcb29d50af0602361d9d6acee04f370f7e01c3674e8" ], "markers": "python_version >= '3.12'", - "version": "==0.12.8" + "version": "==0.12.9" }, "mitmproxy-rs": { "hashes": [ - "sha256:14ea236d0950ab35d667b78b5fe15d43e7345e166e22144624a1283edc78443e", - "sha256:16afd0fc1a00d586ffe2027d217908c3e0389d7d0897eccda6e59fda991e89ba", - "sha256:739591f696cf29913302a72fa9644cf97228774604304a2ea3987fe5588d231c", - "sha256:b0ead519f5a4ab019e7912544c0642f28f8336036ef1480e42a772a8cc947550", - "sha256:c5b0799808a4de0ee60e8f350043820ad56eea738ce3ce25d5c6faaa245b6c9a" + "sha256:1fb9fb4aac9ecb82e2c3c5c439ef5e4961be7934d80ade5e9a99c0a944b8ea2f", + "sha256:1fd716e87da8be3c62daa4325a5ff42bedd951fb8614c5f66caa94b7c21e2593", + "sha256:245922663440330c4b5a36d0194ed559b1dbd5e38545db2eb947180ed12a5e92", + "sha256:afeb3a2da2bc26474e1a2febaea4432430c5fde890dfce33bc4c1e65e6baef1b", + "sha256:c6ffc35c002c675cac534442d92d1cdebd66fafd63754ad33b92ae968ea6e449" ], "index": "pypi", "markers": "python_version >= '3.12'", - "version": "==0.12.8" + "version": "==0.12.9" }, "msgpack": { "hashes": [ @@ -753,11 +765,11 @@ }, "packaging": { "hashes": [ - "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", - "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", + "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" ], "markers": "python_version >= '3.8'", - "version": "==25.0" + "version": "==26.0" }, "pluggy": { "hashes": [ @@ -769,12 +781,12 @@ }, "psycopg": { "hashes": [ - "sha256:3e94bc5f4690247d734599af56e51bae8e0db8e4311ea413f801fef82b14a99b", - "sha256:707a67975ee214d200511177a6a80e56e654754c9afca06a7194ea6bbfde9ca7" + "sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9", + "sha256:f96525a72bcfade6584ab17e89de415ff360748c766f0106959144dcbb38c698" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==3.3.2" + "version": "==3.3.3" }, "publicsuffix2": { "hashes": [ @@ -785,11 +797,11 @@ }, "pyasn1": { "hashes": [ - "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", - "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034" + "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", + "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b" ], "markers": "python_version >= '3.8'", - "version": "==0.6.1" + "version": "==0.6.2" }, "pyasn1-modules": { "hashes": [ @@ -801,11 +813,11 @@ }, "pycparser": { "hashes": [ - "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", - "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934" + "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", + "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" ], - "markers": "python_version >= '3.8'", - "version": "==2.23" + "markers": "python_version >= '3.10'", + "version": "==3.0" }, "pygments": { "hashes": [ @@ -835,11 +847,11 @@ }, "pyopenssl": { "hashes": [ - "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab", - "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b" + "sha256:1fda6fc034d5e3d179d39e59c1895c9faeaf40a79de5fc4cbbfbe0d36f4a77b6", + "sha256:c981cb0a3fd84e8602d7afc209522773b94c1c2446a3c710a75b06fe1beae329" ], "markers": "python_version >= '3.7'", - "version": "==25.1.0" + "version": "==25.3.0" }, "pyparsing": { "hashes": [ @@ -1109,20 +1121,20 @@ }, "wcwidth": { "hashes": [ - "sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605", - "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1" + "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", + "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159" ], - "markers": "python_version >= '3.6'", - "version": "==0.2.14" + "markers": "python_version >= '3.8'", + "version": "==0.6.0" }, "werkzeug": { "hashes": [ - "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905", - "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e" + "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", + "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==3.1.4" + "version": "==3.1.5" }, "wsproto": { "hashes": [ @@ -1305,12 +1317,12 @@ }, "isort": { "hashes": [ - "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", - "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187" + "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d", + "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75" ], "index": "pypi", "markers": "python_full_version >= '3.10.0'", - "version": "==7.0.0" + "version": "==8.0.1" }, "mccabe": { "hashes": [ @@ -1330,27 +1342,27 @@ }, "packaging": { "hashes": [ - "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", - "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", + "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" ], "markers": "python_version >= '3.8'", - "version": "==25.0" + "version": "==26.0" }, "pathspec": { "hashes": [ - "sha256:62f8558917908d237d399b9b338ef455a814801a4688bc41074b25feefd93472", - "sha256:fa32b1eb775ed9ba8d599b22c5f906dc098113989da2c00bf8b210078ca7fb92" + "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", + "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723" ], "markers": "python_version >= '3.9'", - "version": "==1.0.2" + "version": "==1.0.4" }, "platformdirs": { "hashes": [ - "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", - "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31" + "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd", + "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291" ], "markers": "python_version >= '3.10'", - "version": "==4.5.1" + "version": "==4.9.2" }, "pycodestyle": { "hashes": [ diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index fa9028e6b2f..676e16ef6a4 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -32,12 +32,12 @@ jobs: pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester" style_checker_image_name: "ghcr.io/citusdata/stylechecker" style_checker_tools_version: "0.8.33" - sql_snapshot_pg_version: "18.1" - image_suffix: "-v181ce3c" - pg16_version: '{ "major": "16", "full": "16.11" }' - pg17_version: '{ "major": "17", "full": "17.7" }' - pg18_version: '{ "major": "18", "full": "18.1" }' - upgrade_pg_versions: "16.11-17.7-18.1" + sql_snapshot_pg_version: "18.3" + image_suffix: "-vac4338a" + pg16_version: '{ "major": "16", "full": "16.13" }' + pg17_version: '{ "major": "17", "full": "17.9" }' + pg18_version: '{ "major": "18", "full": "18.3" }' + upgrade_pg_versions: "16.13-17.9-18.3" steps: # Since GHA jobs need at least one step we use a noop step here. - name: Set up parameters diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c index b53b8cd1ac5..acb412291de 100644 --- a/src/backend/distributed/commands/alter_table.c +++ b/src/backend/distributed/commands/alter_table.c @@ -1354,6 +1354,7 @@ CreateTableConversion(TableConversionParameters *params) } + Oid relam = relation->rd_rel->relam; relation_close(relation, NoLock); con->distributionKey = BuildDistributionKeyFromColumnName(con->relationId, con->distributionColumn, @@ -1363,11 +1364,11 @@ CreateTableConversion(TableConversionParameters *params) if (!PartitionedTable(con->relationId) && !IsForeignTable(con->relationId)) { HeapTuple amTuple = SearchSysCache1(AMOID, ObjectIdGetDatum( - relation->rd_rel->relam)); + relam)); if (!HeapTupleIsValid(amTuple)) { ereport(ERROR, (errmsg("cache lookup failed for access method %d", - relation->rd_rel->relam))); + relam))); } Form_pg_am amForm = (Form_pg_am) GETSTRUCT(amTuple); con->originalAccessMethod = NameStr(amForm->amname); diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index e75623d54f2..610bbddc59c 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -175,6 +175,7 @@ PreprocessIndexStmt(Node *node, const char *createIndexCommand, namespaceName); } + Oid relationOid = relation->rd_id; table_close(relation, NoLock); Oid relationId = CreateIndexStmtGetRelationId(createIndexStatement); @@ -199,7 +200,7 @@ PreprocessIndexStmt(Node *node, const char *createIndexCommand, * it on a copy not to interfere with standard process utility. */ IndexStmt *copyCreateIndexStatement = - transformIndexStmt(relation->rd_id, copyObject(createIndexStatement), + transformIndexStmt(relationOid, copyObject(createIndexStatement), createIndexCommand); /* ensure we copy string into proper context */ diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index f352b83930f..085b194c36a 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -15,11 +15,15 @@ #include "catalog/namespace.h" #include "lib/stringinfo.h" #include "nodes/nodes.h" +#include "parser/parse_expr.h" #include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/ruleutils.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/relay_utility.h" @@ -231,6 +235,42 @@ AppendStatTypes(StringInfo buf, CreateStatsStmt *stmt) } +/* See ruleutils.c in postgres for the logic here. */ +static bool +looks_like_function(Node *node) +{ + if (node == NULL) + { + return false; /* probably shouldn't happen */ + } + switch (nodeTag(node)) + { + case T_FuncExpr: + { + /* OK, unless it's going to deparse as a cast */ + return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL || + ((FuncExpr *) node)->funcformat == COERCE_SQL_SYNTAX); + } + + case T_NullIfExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + { + /* these are all accepted by func_expr_common_subexpr */ + return true; + } + + default: + { + break; + } + } + return false; +} + + static void AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) { @@ -240,15 +280,54 @@ AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) { if (!column->name) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only simple column references are allowed " - "in CREATE STATISTICS"))); + /* + * Since these expressions are parser statements, we first call + * transform to get the transformed Expr tree, and then deparse + * the transformed tree. This is similar to the logic found in + * deparse_table_stmts for check constraints. + */ + if (list_length(stmt->relations) != 1) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "cannot use expressions in CREATE STATISTICS with multiple relations"))); + } + + RangeVar *rel = (RangeVar *) linitial(stmt->relations); + bool missingOk = false; + Oid relOid = RangeVarGetRelid(rel, AccessShareLock, missingOk); + + /* Add table name to the name space in parse state. Otherwise column names + * cannot be found. + */ + Relation relation = table_open(relOid, AccessShareLock); + ParseState *pstate = make_parsestate(NULL); + AddRangeTableEntryToQueryCompat(pstate, relation); + Node *exprCooked = transformExpr(pstate, column->expr, + EXPR_KIND_STATS_EXPRESSION); + + char *relationName = get_rel_name(relOid); + List *relationCtx = deparse_context_for(relationName, relOid); + + char *exprSql = deparse_expression(exprCooked, relationCtx, false, false); + relation_close(relation, NoLock); + + /* Need parens if it's not a bare function call */ + if (looks_like_function(exprCooked)) + { + appendStringInfoString(buf, exprSql); + } + else + { + appendStringInfo(buf, "(%s)", exprSql); + } } + else + { + const char *columnName = quote_identifier(column->name); - const char *columnName = quote_identifier(column->name); - - appendStringInfoString(buf, columnName); + appendStringInfoString(buf, columnName); + } if (column != llast(stmt->exprs)) { diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 6a25005a111..f9b2574ad6e 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -64,6 +64,7 @@ int LimitClauseRowFetchCount = -1; /* number of rows to fetch from each task */ double CountDistinctErrorRate = 0.0; /* precision of count(distinct) approximate */ int CoordinatorAggregationStrategy = COORDINATOR_AGGREGATION_ROW_GATHER; +bool AllowAggregateWorkerCombineOnInternalTypes = true; /* Constant used throughout file */ static const uint32 masterTableId = 1; /* first range table reference on the master node */ @@ -281,7 +282,7 @@ static Oid CitusFunctionOidWithSignature(char *functionName, int numargs, Oid *a static Oid WorkerPartialAggOid(void); static Oid WorkerBinaryPartialAggOid(void); static Oid CoordBinaryCombineAggOid(void); -static bool IsTypeBinarySerializable(Oid transitionType); +static bool IsAggTransTypeBinarySerializable(Form_pg_aggregate aggForm); static Oid CoordCombineAggOid(void); static Oid AggregateFunctionOid(const char *functionName, Oid inputType); static Oid TypeOid(Oid schemaId, const char *typeName); @@ -2131,7 +2132,7 @@ MasterAggregateExpression(Aggref *originalAggregate, aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple); combine = aggform->aggcombinefn; useBinaryCoordinatorCombine = aggform->aggtranstype != InvalidOid && - IsTypeBinarySerializable(aggform->aggtranstype); + IsAggTransTypeBinarySerializable(aggform); ReleaseSysCache(aggTuple); } @@ -3296,7 +3297,7 @@ WorkerAggregateExpressionList(Aggref *originalAggregate, aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple); combine = aggform->aggcombinefn; useBinaryWorkerAggregate = (OidIsValid(aggform->aggtranstype) && - IsTypeBinarySerializable(aggform->aggtranstype)); + IsAggTransTypeBinarySerializable(aggform)); ReleaseSysCache(aggTuple); } @@ -3568,6 +3569,18 @@ AggregateEnabledCustom(Aggref *aggregateExpression) bool supportsSafeCombine = typeform->typtype != TYPTYPE_PSEUDO; + if (AllowAggregateWorkerCombineOnInternalTypes && + typeform->oid == INTERNALOID && !supportsSafeCombine) + { + /* check if the type supports a SERIALFUNC/DESERIALFUNC - if it does + * then we can leverage that for safe transfer of the state across the wire. + */ + if (aggform->aggserialfn != InvalidOid && aggform->aggdeserialfn != InvalidOid) + { + supportsSafeCombine = true; + } + } + ReleaseSysCache(aggTuple); ReleaseSysCache(typeTuple); @@ -3848,8 +3861,20 @@ TypeOid(Oid schemaId, const char *typeName) static bool -IsTypeBinarySerializable(Oid transitionType) +IsAggTransTypeBinarySerializable(Form_pg_aggregate aggForm) { + Oid transitionType = aggForm->aggtranstype; + + if (AllowAggregateWorkerCombineOnInternalTypes && + transitionType == INTERNALOID) + { + /* For aggregates with internal transition types, we apply the binary serialization + * check on the output value of the SERIALFUNC. If a serialfunc exists, Postgres + * requires that the serialfunc return a bytea - which will be binary serializable + */ + return (aggForm->aggserialfn != InvalidOid); + } + HeapTuple typeTuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(transitionType)); if (!HeapTupleIsValid(typeTuple)) { diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index e3e31a5f036..9aaa75095f0 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -984,6 +984,22 @@ RegisterCitusConfigVariables(void) GUC_STANDARD, NULL, NULL, NULL); + DefineCustomBoolVariable( + "citus.allow_aggregate_worker_combine_on_internal_types", + gettext_noop("Enables aggregate worker partial aggregates on aggregates that " + "have internal type for the aggregate partial state storage."), + gettext_noop( + "This setting allows the use of pushdown of custom aggregates that have " + "an STYPE that is internal. This is typically okay to do, but if a custom aggregate " + "persists OID information or any node specific data into the state, this can cause " + "weirdness when combining in the coordinator, so this is left as an option to turn off " + "in those cases worker combine functions on internal types."), + &AllowAggregateWorkerCombineOnInternalTypes, + true, + PGC_USERSET, + GUC_STANDARD, + NULL, NULL, NULL); + DefineCustomBoolVariable( "citus.allow_modifications_from_workers_to_replicated_tables", gettext_noop("Enables modifications from workers to replicated " diff --git a/src/backend/distributed/utils/aggregate_utils.c b/src/backend/distributed/utils/aggregate_utils.c index e6354e4875a..df85bf9da6d 100644 --- a/src/backend/distributed/utils/aggregate_utils.c +++ b/src/backend/distributed/utils/aggregate_utils.c @@ -623,7 +623,8 @@ GetAggregateTransitionType(StypeBox *box) "worker_partial_agg_ffunc expects an aggregate with COMBINEFUNC"))); } - if (aggform->aggtranstype == INTERNALOID) + if (aggform->aggtranstype == INTERNALOID && + aggform->aggserialfn == InvalidOid) { ereport(ERROR, (errmsg( @@ -660,6 +661,40 @@ WorkerPartialAggregateApplyFFunc(PG_FUNCTION_ARGS) } +/* If the transtype is internal, we need to use the SERIALFUNC of the aggregate + * to serialize the value in the worker. The COMBINEFUNC on the coordinator will + * then use the DESERIALFUNC to deserialize the value. + */ +static Datum +CheckAndCallSerialFunc(PG_FUNCTION_ARGS, StypeBox *box, bool *outputIsNull) +{ + LOCAL_FCINFO(serialFcInfo, 1); + FmgrInfo serialInfo; + Form_pg_aggregate aggform; + HeapTuple aggtuple = GetAggregateForm(box->agg, &aggform); + + if (aggform->aggserialfn == InvalidOid) + { + ereport(ERROR, + (errmsg( + "worker_partial_agg_ffunc expects aggregates with internal transition state to have a SERIALFUNC"))); + } + + /* Otherwise, first invoke the serialfunc to ensure that we produce a serialiable value from the boxValue */ + Assert(get_func_rettype(aggform->aggserialfn) == BYTEAOID); + fmgr_info(aggform->aggserialfn, &serialInfo); + + InitFunctionCallInfoData(*serialFcInfo, &serialInfo, 1, fcinfo->fncollation, + fcinfo->context, fcinfo->resultinfo); + fcSetArgExt(serialFcInfo, 0, box->value, box->valueNull); + + Datum result = FunctionCallInvoke(serialFcInfo); + *outputIsNull = serialFcInfo->isnull; + ReleaseSysCache(aggtuple); + return result; +} + + /* * worker_partial_agg_ffunc serializes transition state, * essentially implementing the following pseudocode: @@ -682,12 +717,21 @@ worker_partial_agg_ffunc(PG_FUNCTION_ARGS) } Oid transtype = GetAggregateTransitionType(box); + + Datum boxValue = box->value; + bool boxValueNull = box->valueNull; + if (transtype == INTERNALOID) + { + ereport(ERROR, (errmsg("worker_partial_agg_ffunc does not support output" + " of aggregates with INTERNAL transition state"))); + } + getTypeOutputInfo(transtype, &typoutput, &typIsVarlena); fmgr_info(typoutput, &info); InitFunctionCallInfoData(*innerFcinfo, &info, 1, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); - fcSetArgExt(innerFcinfo, 0, box->value, box->valueNull); + fcSetArgExt(innerFcinfo, 0, boxValue, boxValueNull); Datum result = FunctionCallInvoke(innerFcinfo); @@ -721,12 +765,24 @@ worker_binary_partial_agg_ffunc(PG_FUNCTION_ARGS) } Oid transtype = GetAggregateTransitionType(box); + + + Datum boxValue = box->value; + bool boxValueNull = box->valueNull; + if (transtype == INTERNALOID) + { + /* Call and store the output of the SERIALFUNC - the output type + * then is always BYTEAOID. */ + boxValue = CheckAndCallSerialFunc(fcinfo, box, &boxValueNull); + transtype = BYTEAOID; + } + getTypeBinaryOutputInfo(transtype, &typoutput, &typIsVarlena); fmgr_info(typoutput, &info); InitFunctionCallInfoData(*innerFcinfo, &info, 1, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); - fcSetArgExt(innerFcinfo, 0, box->value, box->valueNull); + fcSetArgExt(innerFcinfo, 0, boxValue, boxValueNull); Datum result = FunctionCallInvoke(innerFcinfo); @@ -738,6 +794,29 @@ worker_binary_partial_agg_ffunc(PG_FUNCTION_ARGS) } +static Datum +DeserializeBoxValue(Oid deserialFunc, Datum value, bool valueNull, + FunctionCallInfo fcinfo, + bool *outputIsNull) +{ + LOCAL_FCINFO(deserialFcInfo, 3); + FmgrInfo deserialInfo; + + fmgr_info(deserialFunc, &deserialInfo); + + InitFunctionCallInfoData(*deserialFcInfo, &deserialInfo, 2, fcinfo->fncollation, + fcinfo->context, fcinfo->resultinfo); + fcSetArgExt(deserialFcInfo, 0, value, valueNull); + + /* Arg1 is not used and is internal */ + fcSetArgExt(deserialFcInfo, 1, (Datum) 0, false); + + Datum result = FunctionCallInvoke(deserialFcInfo); + *outputIsNull = deserialFcInfo->isnull; + return result; +} + + static Datum CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) { @@ -745,6 +824,7 @@ CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) FmgrInfo info; Form_pg_aggregate aggform; Form_pg_type transtypeform; + Oid deserialFunc = InvalidOid; Datum value; StypeBox *box = NULL; @@ -769,9 +849,16 @@ CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) if (aggform->aggtranstype == INTERNALOID) { - ereport(ERROR, - (errmsg( - "coord_combine_agg_sfunc does not support aggregates with INTERNAL transition state"))); + if (aggform->aggdeserialfn == InvalidOid) + { + ereport(ERROR, + (errmsg( + "coord_combine_agg_sfunc does not support aggregates with INTERNAL transition state"))); + } + else + { + deserialFunc = aggform->aggdeserialfn; + } } Oid combine = aggform->aggcombinefn; @@ -791,10 +878,21 @@ CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) } bool valueNull = PG_ARGISNULL(2); - HeapTuple transtypetuple = GetTypeForm(box->transtype, &transtypeform); - Oid ioparam = getTypeIOParam(transtypetuple); + + /* If the stype is internal, this needs to through first + * deserializing the wire output to the intermediate state. + */ + Oid deserializationType = box->transtype; + if (box->transtype == INTERNALOID) + { + /* For a deserialfunc, the input is a BYTEAOID */ + deserializationType = BYTEAOID; + } + + HeapTuple deserializationTypeTuple = GetTypeForm(deserializationType, &transtypeform); + Oid ioparam = getTypeIOParam(deserializationTypeTuple); Oid deserial = isBinaryInput ? transtypeform->typreceive : transtypeform->typinput; - ReleaseSysCache(transtypetuple); + ReleaseSysCache(deserializationTypeTuple); fmgr_info(deserial, &info); if (valueNull && info.fn_strict) @@ -864,6 +962,15 @@ CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) valueNull = innerFcinfo->isnull; } + /* If the stype is internal, we need to go through one additional step + * of now calling the deserialfunc to go from the serialized type to + * internal before we call the combine function. + */ + if (box->transtype == INTERNALOID && !valueNull) + { + value = DeserializeBoxValue(deserialFunc, value, valueNull, fcinfo, &valueNull); + } + fmgr_info(combine, &info); if (info.fn_strict) diff --git a/src/backend/distributed/utils/multi_partitioning_utils.c b/src/backend/distributed/utils/multi_partitioning_utils.c index 56794825e4e..d801cf28f08 100644 --- a/src/backend/distributed/utils/multi_partitioning_utils.c +++ b/src/backend/distributed/utils/multi_partitioning_utils.c @@ -335,11 +335,12 @@ FixPartitionShardIndexNames(Oid relationId, Oid parentIndexOid) } else { + char *relname = pstrdup(RelationGetRelationName(relation)); relation_close(relation, NoLock); ereport(ERROR, (errmsg("Fixing shard index names is only applicable to " "partitioned tables or partitions, " "and \"%s\" is neither", - RelationGetRelationName(relation)))); + relname))); } CreateFixPartitionShardIndexNames(parentRelationId, diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index 17fe57af029..9ffe70bd429 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -426,7 +426,17 @@ LockShardListMetadataOnWorkers(LOCKMODE lockmode, List *shardIntervalList) appendStringInfo(lockCommand, "])"); - SendCommandToWorkersWithMetadata(lockCommand->data); + /* + * Disable idle_in_transaction_session_timeout on metadata workers before + * acquiring locks. In block_writes mode, these connections stay open for + * the entire shard copy which can take hours for large shards. Without + * this, the timeout would kill the connection and fail the move. + * SET LOCAL scopes the change to this transaction only. + */ + List *commandList = list_make2( + "SET LOCAL idle_in_transaction_session_timeout = 0", + lockCommand->data); + SendCommandListToWorkersWithMetadata(commandList); } diff --git a/src/backend/distributed/utils/tenant_schema_metadata.c b/src/backend/distributed/utils/tenant_schema_metadata.c index 57ae1d15199..170a35f1ff1 100644 --- a/src/backend/distributed/utils/tenant_schema_metadata.c +++ b/src/backend/distributed/utils/tenant_schema_metadata.c @@ -14,8 +14,10 @@ #include "access/genam.h" #include "access/htup.h" #include "access/table.h" +#include "catalog/pg_namespace_d.h" #include "storage/lockdefs.h" #include "utils/fmgroids.h" +#include "utils/lsyscache.h" #include "utils/relcache.h" #include "distributed/colocation_utils.h" @@ -32,30 +34,38 @@ bool IsTenantSchema(Oid schemaId) { /* - * We don't allow creating tenant schemas when there is a version - * mismatch. Even more, SchemaIdGetTenantColocationId() would throw an - * error if the underlying pg_dist_schema metadata table has not - * been created yet, which is the case in older versions. For this reason, - * it's safe to assume that it cannot be a tenant schema when there is a - * version mismatch. + * Bail out early if there is a version mismatch between the Citus + * shared library and the installed SQL extension. This prevents + * querying pg_dist_schema when the catalog table may not exist + * (e.g., during multi_extension tests that install versions older + * than 12.0). * - * But it's a bit tricky that we do the same when version checks are - * disabled because then CheckCitusVersion() returns true even if there - * is a version mismatch. And in that case, the tests that are trying to - * create tables (in multi_extension.sql) in older versions would - * fail when deciding whether we should create a tenant table or not. + * When version checks are disabled (citus.enable_version_checks=off), + * CheckCitusVersion() returns true unconditionally, even if there is + * a real version mismatch. In that case, we need to verify that + * pg_dist_schema actually exists before querying it, since the + * multi_extension test installs old versions (e.g. 8.0-1) with + * version checks disabled and creates tables in that state. * - * The downside of doing so is that, for example, we will skip deleting - * the tenant schema entry from pg_dist_schema when dropping a - * tenant schema while the version checks are disabled even if there was - * no version mismatch. But we're okay with that because we don't expect - * users to disable version checks anyway. + * On the normal path (version checks enabled, versions compatible), + * this adds no overhead -- the !EnableVersionChecks branch is skipped + * entirely. */ - if (!EnableVersionChecks || !CheckCitusVersion(DEBUG4)) + if (!CheckCitusVersion(DEBUG4)) { return false; } + if (!EnableVersionChecks) + { + Oid distSchemaOid = get_relname_relid("pg_dist_schema", + PG_CATALOG_NAMESPACE); + if (!OidIsValid(distSchemaOid)) + { + return false; + } + } + return SchemaIdGetTenantColocationId(schemaId) != INVALID_COLOCATION_ID; } diff --git a/src/include/distributed/query_pushdown_planning.h b/src/include/distributed/query_pushdown_planning.h index 87ff07aeb21..0b69d36c75f 100644 --- a/src/include/distributed/query_pushdown_planning.h +++ b/src/include/distributed/query_pushdown_planning.h @@ -22,6 +22,7 @@ /* Config variables managed via guc.c */ extern bool SubqueryPushdown; extern int ValuesMaterializationThreshold; +extern bool AllowAggregateWorkerCombineOnInternalTypes; extern bool CanPushdownSubquery(Query *subqueryTree, bool outerMostQueryHasLimit); diff --git a/src/test/regress/Pipfile b/src/test/regress/Pipfile index 8ade5491338..06e820beb44 100644 --- a/src/test/regress/Pipfile +++ b/src/test/regress/Pipfile @@ -16,7 +16,7 @@ tornado = ">=6.5.1,<6.6.0" zstandard = ">=0.25.0" construct = "*" docopt = "==0.6.2" -cryptography = "==44.0.3" +cryptography = "==46.0.5" pytest = "*" psycopg = "*" filelock = "*" @@ -25,7 +25,7 @@ pytest-timeout = "*" pytest-xdist = "*" pytest-repeat = "*" pyyaml = "*" -werkzeug = "==3.1.4" +werkzeug = "==3.1.5" "typing-extensions" = ">=4.13.2,<5" pyperclip = "==1.9.0" diff --git a/src/test/regress/Pipfile.lock b/src/test/regress/Pipfile.lock index 5b662d4343f..8d7fadb409d 100644 --- a/src/test/regress/Pipfile.lock +++ b/src/test/regress/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5f734fff88a49010712613a14addae5f38347c691d0b3004b09f59ba5c7cc061" + "sha256": "92e863acf1c77d535dc7da881b4e4cf03bb8fa748769c845aaee27d86d94657c" }, "pipfile-spec": 6, "requires": { @@ -292,11 +292,11 @@ }, "certifi": { "hashes": [ - "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", - "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120" + "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", + "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" ], "markers": "python_version >= '3.7'", - "version": "==2026.1.4" + "version": "==2026.2.25" }, "cffi": { "hashes": [ @@ -407,47 +407,59 @@ }, "cryptography": { "hashes": [ - "sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259", - "sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43", - "sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645", - "sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8", - "sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44", - "sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d", - "sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f", - "sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d", - "sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54", - "sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9", - "sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137", - "sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f", - "sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c", - "sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334", - "sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c", - "sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b", - "sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2", - "sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375", - "sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88", - "sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5", - "sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647", - "sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c", - "sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359", - "sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5", - "sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d", - "sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028", - "sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01", - "sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904", - "sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d", - "sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93", - "sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06", - "sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff", - "sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76", - "sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff", - "sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759", - "sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4", - "sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053" + "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", + "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", + "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9", + "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356", + "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257", + "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", + "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", + "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c", + "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", + "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", + "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", + "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", + "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", + "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", + "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", + "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4", + "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4", + "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", + "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", + "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", + "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678", + "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", + "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", + "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0", + "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", + "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d", + "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", + "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", + "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981", + "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", + "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", + "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2", + "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", + "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663", + "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", + "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", + "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", + "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a", + "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", + "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", + "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a", + "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826", + "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee", + "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", + "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648", + "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da", + "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", + "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", + "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87" ], "index": "pypi", - "markers": "python_version >= '3.7' and python_full_version not in '3.9.0, 3.9.1'", - "version": "==44.0.3" + "markers": "python_version >= '3.8' and python_full_version not in '3.9.0, 3.9.1'", + "version": "==46.0.5" }, "docopt": { "hashes": [ @@ -466,12 +478,12 @@ }, "filelock": { "hashes": [ - "sha256:a2241ff4ddde2a7cebddf78e39832509cb045d18ec1a09d7248d6bfc6bfbbe64", - "sha256:fbba7237d6ea277175a32c54bb71ef814a8546d8601269e1bfc388de333974e8" + "sha256:5ccf8069f7948f494968fc0713c10e5c182a9c9d9eef3a636307a20c2490f047", + "sha256:8f00faf3abf9dc730a1ffe9c354ae5c04e079ab7d3a683b7c32da5dd05f26af3" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==3.20.2" + "version": "==3.25.0" }, "flask": { "hashes": [ @@ -664,24 +676,24 @@ }, "mitmproxy-linux": { "hashes": [ - "sha256:0bea9353c71ebfd2174f6730b3fd0fdff3adea1aa15450035bed3b83e36ef455", - "sha256:2238455e65970382825baed2e998601ea82d8dcaae51bd8ee0859d596524a822", - "sha256:fbcb25316e95d0b2b5ced4e0cc3d90fdb1b7169300a005cc79339894d665363a" + "sha256:94b10fee02aa42287739623cef921e1a53955005d45c9e2fa309ae9f0bf8d37d", + "sha256:b4413e27c692f30036ad6d73432826e728ede026fac8e51651d0c545dd0177f2", + "sha256:ee842865a05f69196004ddcb29d50af0602361d9d6acee04f370f7e01c3674e8" ], "markers": "python_version >= '3.12'", - "version": "==0.12.8" + "version": "==0.12.9" }, "mitmproxy-rs": { "hashes": [ - "sha256:14ea236d0950ab35d667b78b5fe15d43e7345e166e22144624a1283edc78443e", - "sha256:16afd0fc1a00d586ffe2027d217908c3e0389d7d0897eccda6e59fda991e89ba", - "sha256:739591f696cf29913302a72fa9644cf97228774604304a2ea3987fe5588d231c", - "sha256:b0ead519f5a4ab019e7912544c0642f28f8336036ef1480e42a772a8cc947550", - "sha256:c5b0799808a4de0ee60e8f350043820ad56eea738ce3ce25d5c6faaa245b6c9a" + "sha256:1fb9fb4aac9ecb82e2c3c5c439ef5e4961be7934d80ade5e9a99c0a944b8ea2f", + "sha256:1fd716e87da8be3c62daa4325a5ff42bedd951fb8614c5f66caa94b7c21e2593", + "sha256:245922663440330c4b5a36d0194ed559b1dbd5e38545db2eb947180ed12a5e92", + "sha256:afeb3a2da2bc26474e1a2febaea4432430c5fde890dfce33bc4c1e65e6baef1b", + "sha256:c6ffc35c002c675cac534442d92d1cdebd66fafd63754ad33b92ae968ea6e449" ], "index": "pypi", "markers": "python_version >= '3.12'", - "version": "==0.12.8" + "version": "==0.12.9" }, "msgpack": { "hashes": [ @@ -753,11 +765,11 @@ }, "packaging": { "hashes": [ - "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", - "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", + "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" ], "markers": "python_version >= '3.8'", - "version": "==25.0" + "version": "==26.0" }, "pluggy": { "hashes": [ @@ -769,12 +781,12 @@ }, "psycopg": { "hashes": [ - "sha256:3e94bc5f4690247d734599af56e51bae8e0db8e4311ea413f801fef82b14a99b", - "sha256:707a67975ee214d200511177a6a80e56e654754c9afca06a7194ea6bbfde9ca7" + "sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9", + "sha256:f96525a72bcfade6584ab17e89de415ff360748c766f0106959144dcbb38c698" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==3.3.2" + "version": "==3.3.3" }, "publicsuffix2": { "hashes": [ @@ -785,11 +797,11 @@ }, "pyasn1": { "hashes": [ - "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", - "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034" + "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", + "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b" ], "markers": "python_version >= '3.8'", - "version": "==0.6.1" + "version": "==0.6.2" }, "pyasn1-modules": { "hashes": [ @@ -801,11 +813,11 @@ }, "pycparser": { "hashes": [ - "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", - "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934" + "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", + "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" ], - "markers": "python_version >= '3.8'", - "version": "==2.23" + "markers": "python_version >= '3.10'", + "version": "==3.0" }, "pygments": { "hashes": [ @@ -835,11 +847,11 @@ }, "pyopenssl": { "hashes": [ - "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab", - "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b" + "sha256:1fda6fc034d5e3d179d39e59c1895c9faeaf40a79de5fc4cbbfbe0d36f4a77b6", + "sha256:c981cb0a3fd84e8602d7afc209522773b94c1c2446a3c710a75b06fe1beae329" ], "markers": "python_version >= '3.7'", - "version": "==25.1.0" + "version": "==25.3.0" }, "pyparsing": { "hashes": [ @@ -1109,20 +1121,20 @@ }, "wcwidth": { "hashes": [ - "sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605", - "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1" + "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", + "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159" ], - "markers": "python_version >= '3.6'", - "version": "==0.2.14" + "markers": "python_version >= '3.8'", + "version": "==0.6.0" }, "werkzeug": { "hashes": [ - "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905", - "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e" + "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", + "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==3.1.4" + "version": "==3.1.5" }, "wsproto": { "hashes": [ @@ -1305,12 +1317,12 @@ }, "isort": { "hashes": [ - "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", - "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187" + "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d", + "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75" ], "index": "pypi", "markers": "python_full_version >= '3.10.0'", - "version": "==7.0.0" + "version": "==8.0.1" }, "mccabe": { "hashes": [ @@ -1330,27 +1342,27 @@ }, "packaging": { "hashes": [ - "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", - "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", + "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" ], "markers": "python_version >= '3.8'", - "version": "==25.0" + "version": "==26.0" }, "pathspec": { "hashes": [ - "sha256:62f8558917908d237d399b9b338ef455a814801a4688bc41074b25feefd93472", - "sha256:fa32b1eb775ed9ba8d599b22c5f906dc098113989da2c00bf8b210078ca7fb92" + "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", + "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723" ], "markers": "python_version >= '3.9'", - "version": "==1.0.2" + "version": "==1.0.4" }, "platformdirs": { "hashes": [ - "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", - "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31" + "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd", + "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291" ], "markers": "python_version >= '3.10'", - "version": "==4.5.1" + "version": "==4.9.2" }, "pycodestyle": { "hashes": [ diff --git a/src/test/regress/expected/aggregate_support.out b/src/test/regress/expected/aggregate_support.out index 78cd54292df..7f0b8c4639c 100644 --- a/src/test/regress/expected/aggregate_support.out +++ b/src/test/regress/expected/aggregate_support.out @@ -294,6 +294,79 @@ EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF, BUFFERS OFF) SELECT key -> Seq Scan on aggdata_83674000 aggdata (actual rows=4 loops=1) (17 rows) +-- aggregates with internal stype works. +CREATE AGGREGATE internalsum(int8) ( + sfunc = int8_avg_accum, + stype = internal, + finalfunc = numeric_poly_sum, + combinefunc = int8_avg_combine, + serialfunc = int8_avg_serialize, + deserialfunc = int8_avg_deserialize +); +CREATE AGGREGATE internalsum_noserial(int8) ( + sfunc = int8_avg_accum, + stype = internal, + finalfunc = numeric_poly_sum, + combinefunc = int8_avg_combine +); +SELECT key, internalsum(val), sum(val) from aggdata group by key order by key; + key | internalsum | sum +--------------------------------------------------------------------- + 1 | 2 | 2 + 2 | 10 | 10 + 3 | 4 | 4 + 5 | | + 6 | | + 7 | 8 | 8 + 9 | 0 | 0 +(7 rows) + +-- see that the explain is pushed to the shards. +EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF, BUFFERS OFF) SELECT key, internalsum(val) from aggdata group by key order by key; + QUERY PLAN +--------------------------------------------------------------------- + Sort (actual rows=7 loops=1) + Sort Key: remote_scan.key + Sort Method: quicksort Memory: 25kB + -> HashAggregate (actual rows=7 loops=1) + Group Key: remote_scan.key + -> Custom Scan (Citus Adaptive) (actual rows=10 loops=1) + Task Count: 4 + Tuple data received from nodes: 290 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 118 bytes + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate (actual rows=4 loops=1) + Group Key: key + -> Seq Scan on aggdata_83674000 aggdata (actual rows=4 loops=1) +(17 rows) + +-- without a serialfunc always fails. +SELECT key, internalsum_noserial(val), sum(val) from aggdata group by key order by key; +ERROR: unsupported aggregate function internalsum_noserial +-- but this works if we allow coordinator combine +set citus.coordinator_aggregation_strategy to 'row-gather'; +SELECT key, internalsum_noserial(val), sum(val) from aggdata group by key order by key; + key | internalsum_noserial | sum +--------------------------------------------------------------------- + 1 | 2 | 2 + 2 | 10 | 10 + 3 | 4 | 4 + 5 | | + 6 | | + 7 | 8 | 8 + 9 | 0 | 0 +(7 rows) + +set citus.coordinator_aggregation_strategy to 'disabled'; +-- if the GUC is unset, do not allow pushdown. +set citus.allow_aggregate_worker_combine_on_internal_types to off; +SELECT key, internalsum(val), sum(val) from aggdata group by key order by key; +ERROR: unsupported aggregate function internalsum +reset citus.allow_aggregate_worker_combine_on_internal_types; +DROP AGGREGATE internalsum(int8); +DROP AGGREGATE internalsum_noserial(int8); -- binary string aggregation create function binstragg_sfunc(s text, e1 text, e2 text) returns text immutable language plpgsql as $$ diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index d29d1bf5a74..ae7b7b0d721 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -208,7 +208,7 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx reindex(TABLESPACE test_tablespace1) index idx; ERROR: tablespace "test_tablespace1" does not exist reset citus.log_remote_commands; --- CREATE STATISTICS only allow simple column references +-- CREATE STATISTICS allows table references too CREATE TABLE tbl1(a timestamp, b int); SELECT create_distributed_table('tbl1','a'); create_distributed_table @@ -216,11 +216,9 @@ SELECT create_distributed_table('tbl1','a'); (1 row) --- the last one should error out CREATE STATISTICS s1 (dependencies) ON a, b FROM tbl1; CREATE STATISTICS s2 (mcv) ON a, b FROM tbl1; CREATE STATISTICS s3 (ndistinct) ON date_trunc('month', a), date_trunc('day', a) FROM tbl1; -ERROR: only simple column references are allowed in CREATE STATISTICS set citus.log_remote_commands to off; -- error out in case of ALTER TABLE .. DETACH PARTITION .. CONCURRENTLY/FINALIZE -- only if it's a distributed partitioned table diff --git a/src/test/regress/expected/propagate_statistics.out b/src/test/regress/expected/propagate_statistics.out index 02563c22ae1..7b82fbc5e0d 100644 --- a/src/test/regress/expected/propagate_statistics.out +++ b/src/test/regress/expected/propagate_statistics.out @@ -53,6 +53,22 @@ SELECT create_distributed_table ('test_stats3','a'); (1 row) +-- test creating custom stats with expressions and distributing it. +CREATE TABLE sc2.test_stats_expr ( + a int, + b int, + c float8 +); +CREATE STATISTICS s_expr ON (a + b / 2) FROM sc2.test_stats_expr; +-- succeeds since we replicate it into the shards. +SELECT create_distributed_table('sc2.test_stats_expr', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- add expression stats on the distributed table should work. +CREATE STATISTICS s_expr_post ON (a - (b * 2)), round(c) FROM sc2.test_stats_expr; -- test dropping statistics CREATE TABLE test_stats4 ( a int, @@ -112,7 +128,7 @@ WHERE stxnamespace IN ( ) AND stxname SIMILAR TO '%\_\d+' ORDER BY stxname ASC; - stxname + stxname --------------------------------------------------------------------- neW'Stat_980096 neW'Stat_980098 @@ -162,22 +178,54 @@ ORDER BY stxname ASC; s2_980058 s2_980060 s2_980062 - s9_980129 - s9_980131 - s9_980133 - s9_980135 - s9_980137 - s9_980139 - s9_980141 - s9_980143 - s9_980145 - s9_980147 - s9_980149 - s9_980151 - s9_980153 - s9_980155 - s9_980157 - s9_980159 + s9_980161 + s9_980163 + s9_980165 + s9_980167 + s9_980169 + s9_980171 + s9_980173 + s9_980175 + s9_980177 + s9_980179 + s9_980181 + s9_980183 + s9_980185 + s9_980187 + s9_980189 + s9_980191 + s_expr_980128 + s_expr_980130 + s_expr_980132 + s_expr_980134 + s_expr_980136 + s_expr_980138 + s_expr_980140 + s_expr_980142 + s_expr_980144 + s_expr_980146 + s_expr_980148 + s_expr_980150 + s_expr_980152 + s_expr_980154 + s_expr_980156 + s_expr_980158 + s_expr_post_980128 + s_expr_post_980130 + s_expr_post_980132 + s_expr_post_980134 + s_expr_post_980136 + s_expr_post_980138 + s_expr_post_980140 + s_expr_post_980142 + s_expr_post_980144 + s_expr_post_980146 + s_expr_post_980148 + s_expr_post_980150 + s_expr_post_980152 + s_expr_post_980154 + s_expr_post_980156 + s_expr_post_980158 st1_new_980064 st1_new_980066 st1_new_980068 @@ -194,23 +242,23 @@ ORDER BY stxname ASC; st1_new_980090 st1_new_980092 st1_new_980094 - stats_xy_980161 - stats_xy_980163 - stats_xy_980165 - stats_xy_980167 - stats_xy_980169 - stats_xy_980171 - stats_xy_980173 - stats_xy_980175 - stats_xy_980177 - stats_xy_980179 - stats_xy_980181 - stats_xy_980183 - stats_xy_980185 - stats_xy_980187 - stats_xy_980189 - stats_xy_980191 -(96 rows) + stats_xy_980193 + stats_xy_980195 + stats_xy_980197 + stats_xy_980199 + stats_xy_980201 + stats_xy_980203 + stats_xy_980205 + stats_xy_980207 + stats_xy_980209 + stats_xy_980211 + stats_xy_980213 + stats_xy_980215 + stats_xy_980217 + stats_xy_980219 + stats_xy_980221 + stats_xy_980223 +(128 rows) SELECT count(DISTINCT stxnamespace) FROM pg_statistic_ext diff --git a/src/test/regress/expected/shard_move_constraints_blocking.out b/src/test/regress/expected/shard_move_constraints_blocking.out index 66dec069e7a..61dbf41ec04 100644 --- a/src/test/regress/expected/shard_move_constraints_blocking.out +++ b/src/test/regress/expected/shard_move_constraints_blocking.out @@ -399,3 +399,87 @@ drop cascades to table "blocking shard Move Fkeys Indexes".reference_table drop cascades to table "blocking shard Move Fkeys Indexes".reference_table_8970028 drop cascades to table "blocking shard Move Fkeys Indexes".index_backed_rep_identity DROP ROLE mx_rebalancer_blocking_role_ent; +-- Test: block_writes shard move succeeds even when workers have a low +-- idle_in_transaction_session_timeout. LockShardListMetadataOnWorkers opens +-- coordinated transactions on ALL metadata workers before the data copy. +-- Workers not involved in the copy sit idle-in-transaction for the entire +-- duration. Without the SET LOCAL override, the timeout would kill those +-- connections and fail the move. +SET citus.next_shard_id TO 8980000; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +CREATE SCHEMA blocking_move_idle_timeout; +SET search_path TO blocking_move_idle_timeout; +-- set a very low idle_in_transaction_session_timeout on all nodes +SELECT 1 FROM run_command_on_all_nodes( + 'ALTER SYSTEM SET idle_in_transaction_session_timeout = ''1s'''); + ?column? +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +SELECT 1 FROM run_command_on_all_nodes('SELECT pg_reload_conf()'); + ?column? +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +-- allow the reload to take effect +SELECT pg_sleep(0.5); + pg_sleep +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE test_move(id int PRIMARY KEY, val text); +SELECT create_distributed_table('test_move', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test_move SELECT i, 'val_' || i FROM generate_series(1, 100) i; +-- move a shard using block_writes; should succeed despite the 1s timeout +SELECT citus_move_shard_placement(8980000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode:='block_writes'); + citus_move_shard_placement +--------------------------------------------------------------------- + +(1 row) + +SELECT public.wait_for_resource_cleanup(); + wait_for_resource_cleanup +--------------------------------------------------------------------- + +(1 row) + +-- verify data integrity after move +SELECT count(*) FROM test_move; + count +--------------------------------------------------------------------- + 100 +(1 row) + +-- cleanup: restore idle_in_transaction_session_timeout +SELECT 1 FROM run_command_on_all_nodes( + 'ALTER SYSTEM RESET idle_in_transaction_session_timeout'); + ?column? +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +SELECT 1 FROM run_command_on_all_nodes('SELECT pg_reload_conf()'); + ?column? +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +DROP SCHEMA blocking_move_idle_timeout CASCADE; +NOTICE: drop cascades to table test_move diff --git a/src/test/regress/sql/aggregate_support.sql b/src/test/regress/sql/aggregate_support.sql index 3962a83b1fa..7482c37efda 100644 --- a/src/test/regress/sql/aggregate_support.sql +++ b/src/test/regress/sql/aggregate_support.sql @@ -144,6 +144,45 @@ select covar_pop(valf,val)::numeric(10,5), covar_samp(valf,val)::numeric(10,5) f set citus.explain_analyze_sort_method to 'taskId'; EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF, BUFFERS OFF) SELECT key, sum2_strict(val) from aggdata group by key order by key; + +-- aggregates with internal stype works. +CREATE AGGREGATE internalsum(int8) ( + sfunc = int8_avg_accum, + stype = internal, + finalfunc = numeric_poly_sum, + combinefunc = int8_avg_combine, + serialfunc = int8_avg_serialize, + deserialfunc = int8_avg_deserialize +); + +CREATE AGGREGATE internalsum_noserial(int8) ( + sfunc = int8_avg_accum, + stype = internal, + finalfunc = numeric_poly_sum, + combinefunc = int8_avg_combine +); + +SELECT key, internalsum(val), sum(val) from aggdata group by key order by key; + +-- see that the explain is pushed to the shards. +EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF, BUFFERS OFF) SELECT key, internalsum(val) from aggdata group by key order by key; + +-- without a serialfunc always fails. +SELECT key, internalsum_noserial(val), sum(val) from aggdata group by key order by key; + +-- but this works if we allow coordinator combine +set citus.coordinator_aggregation_strategy to 'row-gather'; +SELECT key, internalsum_noserial(val), sum(val) from aggdata group by key order by key; +set citus.coordinator_aggregation_strategy to 'disabled'; + +-- if the GUC is unset, do not allow pushdown. +set citus.allow_aggregate_worker_combine_on_internal_types to off; +SELECT key, internalsum(val), sum(val) from aggdata group by key order by key; +reset citus.allow_aggregate_worker_combine_on_internal_types; + +DROP AGGREGATE internalsum(int8); +DROP AGGREGATE internalsum_noserial(int8); + -- binary string aggregation create function binstragg_sfunc(s text, e1 text, e2 text) returns text immutable language plpgsql as $$ diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index c12a8c4fa6a..aa9a30fa257 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -42,10 +42,9 @@ reindex(verbose, TABLESPACE test_tablespace) index idx ; -- should error saying table space doesn't exist reindex(TABLESPACE test_tablespace1) index idx; reset citus.log_remote_commands; --- CREATE STATISTICS only allow simple column references +-- CREATE STATISTICS allows table references too CREATE TABLE tbl1(a timestamp, b int); SELECT create_distributed_table('tbl1','a'); --- the last one should error out CREATE STATISTICS s1 (dependencies) ON a, b FROM tbl1; CREATE STATISTICS s2 (mcv) ON a, b FROM tbl1; CREATE STATISTICS s3 (ndistinct) ON date_trunc('month', a), date_trunc('day', a) FROM tbl1; diff --git a/src/test/regress/sql/propagate_statistics.sql b/src/test/regress/sql/propagate_statistics.sql index 7e1f2fa1822..2e2fecfe7a1 100644 --- a/src/test/regress/sql/propagate_statistics.sql +++ b/src/test/regress/sql/propagate_statistics.sql @@ -44,6 +44,20 @@ CREATE SCHEMA sc2; CREATE STATISTICS sc2."neW'Stat" ON a,b FROM test_stats3; SELECT create_distributed_table ('test_stats3','a'); +-- test creating custom stats with expressions and distributing it. +CREATE TABLE sc2.test_stats_expr ( + a int, + b int, + c float8 +); +CREATE STATISTICS s_expr ON (a + b / 2) FROM sc2.test_stats_expr; + +-- succeeds since we replicate it into the shards. +SELECT create_distributed_table('sc2.test_stats_expr', 'a'); + +-- add expression stats on the distributed table should work. +CREATE STATISTICS s_expr_post ON (a - (b * 2)), round(c) FROM sc2.test_stats_expr; + -- test dropping statistics CREATE TABLE test_stats4 ( a int, diff --git a/src/test/regress/sql/shard_move_constraints_blocking.sql b/src/test/regress/sql/shard_move_constraints_blocking.sql index 66b58f42b9c..acbaca76ab9 100644 --- a/src/test/regress/sql/shard_move_constraints_blocking.sql +++ b/src/test/regress/sql/shard_move_constraints_blocking.sql @@ -222,3 +222,41 @@ ALTER TABLE sensors_2020_01_01 DROP CONSTRAINT fkey_from_child_to_child; \c - postgres - :master_port DROP SCHEMA "blocking shard Move Fkeys Indexes" CASCADE; DROP ROLE mx_rebalancer_blocking_role_ent; + +-- Test: block_writes shard move succeeds even when workers have a low +-- idle_in_transaction_session_timeout. LockShardListMetadataOnWorkers opens +-- coordinated transactions on ALL metadata workers before the data copy. +-- Workers not involved in the copy sit idle-in-transaction for the entire +-- duration. Without the SET LOCAL override, the timeout would kill those +-- connections and fail the move. +SET citus.next_shard_id TO 8980000; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; + +CREATE SCHEMA blocking_move_idle_timeout; +SET search_path TO blocking_move_idle_timeout; + +-- set a very low idle_in_transaction_session_timeout on all nodes +SELECT 1 FROM run_command_on_all_nodes( + 'ALTER SYSTEM SET idle_in_transaction_session_timeout = ''1s'''); +SELECT 1 FROM run_command_on_all_nodes('SELECT pg_reload_conf()'); +-- allow the reload to take effect +SELECT pg_sleep(0.5); + +CREATE TABLE test_move(id int PRIMARY KEY, val text); +SELECT create_distributed_table('test_move', 'id'); +INSERT INTO test_move SELECT i, 'val_' || i FROM generate_series(1, 100) i; + +-- move a shard using block_writes; should succeed despite the 1s timeout +SELECT citus_move_shard_placement(8980000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode:='block_writes'); +SELECT public.wait_for_resource_cleanup(); + +-- verify data integrity after move +SELECT count(*) FROM test_move; + +-- cleanup: restore idle_in_transaction_session_timeout +SELECT 1 FROM run_command_on_all_nodes( + 'ALTER SYSTEM RESET idle_in_transaction_session_timeout'); +SELECT 1 FROM run_command_on_all_nodes('SELECT pg_reload_conf()'); + +DROP SCHEMA blocking_move_idle_timeout CASCADE;