From 364beed8912aed013b7253d75479e622a4456a87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd=20Rei=C3=9F?= Date: Mon, 16 Feb 2026 12:13:38 +0100 Subject: [PATCH 1/7] Fixed RELCACHE_FORCE_RELEASE related use-after-free bugs (#8477) See the linked PR for more details. --- src/backend/distributed/commands/alter_table.c | 5 +++-- src/backend/distributed/commands/index.c | 3 ++- src/backend/distributed/utils/multi_partitioning_utils.c | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/backend/distributed/commands/alter_table.c b/src/backend/distributed/commands/alter_table.c index 433bb0fe386..c919c5ced97 100644 --- a/src/backend/distributed/commands/alter_table.c +++ b/src/backend/distributed/commands/alter_table.c @@ -1354,6 +1354,7 @@ CreateTableConversion(TableConversionParameters *params) } + Oid relam = relation->rd_rel->relam; relation_close(relation, NoLock); con->distributionKey = BuildDistributionKeyFromColumnName(con->relationId, con->distributionColumn, @@ -1363,11 +1364,11 @@ CreateTableConversion(TableConversionParameters *params) if (!PartitionedTable(con->relationId) && !IsForeignTable(con->relationId)) { HeapTuple amTuple = SearchSysCache1(AMOID, ObjectIdGetDatum( - relation->rd_rel->relam)); + relam)); if (!HeapTupleIsValid(amTuple)) { ereport(ERROR, (errmsg("cache lookup failed for access method %d", - relation->rd_rel->relam))); + relam))); } Form_pg_am amForm = (Form_pg_am) GETSTRUCT(amTuple); con->originalAccessMethod = NameStr(amForm->amname); diff --git a/src/backend/distributed/commands/index.c b/src/backend/distributed/commands/index.c index 0538f51b3b7..475b3301bdf 100644 --- a/src/backend/distributed/commands/index.c +++ b/src/backend/distributed/commands/index.c @@ -173,6 +173,7 @@ PreprocessIndexStmt(Node *node, const char *createIndexCommand, namespaceName); } + Oid relationOid = relation->rd_id; table_close(relation, NoLock); Oid relationId = CreateIndexStmtGetRelationId(createIndexStatement); @@ -197,7 +198,7 @@ PreprocessIndexStmt(Node *node, const char *createIndexCommand, * it on a copy not to interfere with standard process utility. */ IndexStmt *copyCreateIndexStatement = - transformIndexStmt(relation->rd_id, copyObject(createIndexStatement), + transformIndexStmt(relationOid, copyObject(createIndexStatement), createIndexCommand); /* ensure we copy string into proper context */ diff --git a/src/backend/distributed/utils/multi_partitioning_utils.c b/src/backend/distributed/utils/multi_partitioning_utils.c index 56794825e4e..d801cf28f08 100644 --- a/src/backend/distributed/utils/multi_partitioning_utils.c +++ b/src/backend/distributed/utils/multi_partitioning_utils.c @@ -335,11 +335,12 @@ FixPartitionShardIndexNames(Oid relationId, Oid parentIndexOid) } else { + char *relname = pstrdup(RelationGetRelationName(relation)); relation_close(relation, NoLock); ereport(ERROR, (errmsg("Fixing shard index names is only applicable to " "partitioned tables or partitions, " "and \"%s\" is neither", - RelationGetRelationName(relation)))); + relname))); } CreateFixPartitionShardIndexNames(parentRelationId, From 546f20661a3bed25ebbd1673bd1753ddba17d3c4 Mon Sep 17 00:00:00 2001 From: eaydingol <60466783+eaydingol@users.noreply.github.com> Date: Tue, 17 Feb 2026 14:06:12 +0300 Subject: [PATCH 2/7] Fix IsTenantSchema returning false when version checks are disabled (#8480) IsTenantSchema checks pg_dist_schema when the version check is disabled. Previously, IsTenantSchema() had a guard that returned false whenever citus.enable_version_checks was off. This was added to protect against accessing pg_dist_schema in multi_extension tests that install old Citus versions (pre-12.0) where the table doesn't exist. However, the guard had an unintended side effect: it completely disabled schema-based sharding during mixed-version testing (CITUSVERSION/N1MODE), even when no actual version mismatch existed. This caused 5 tests to fail (single_node, schema_based_sharding, citus_schema_distribute_undistribute, citus_schema_move, local_shard_utility_command_execution) because tables created in tenant schemas were not being distributed. The root cause was an asymmetry: CREATE SCHEMA correctly registered the tenant schema in pg_dist_schema (ShouldUseSchemaBasedSharding has no version check guard), but CREATE TABLE in that schema silently fell through to a local table because IsTenantSchema returned false. CheckCitusVersion(). When version checks are disabled, CheckCitusVersion() returns true unconditionally, which is safe because: - pg_dist_schema exists in all supported Citus versions (>= 12.0) - mixed-version tests never install versions old enough to lack it - multi_extension (which installs pre-12.0 versions) is not run in mixed-version scenarios --- .../utils/tenant_schema_metadata.c | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/src/backend/distributed/utils/tenant_schema_metadata.c b/src/backend/distributed/utils/tenant_schema_metadata.c index 57ae1d15199..170a35f1ff1 100644 --- a/src/backend/distributed/utils/tenant_schema_metadata.c +++ b/src/backend/distributed/utils/tenant_schema_metadata.c @@ -14,8 +14,10 @@ #include "access/genam.h" #include "access/htup.h" #include "access/table.h" +#include "catalog/pg_namespace_d.h" #include "storage/lockdefs.h" #include "utils/fmgroids.h" +#include "utils/lsyscache.h" #include "utils/relcache.h" #include "distributed/colocation_utils.h" @@ -32,30 +34,38 @@ bool IsTenantSchema(Oid schemaId) { /* - * We don't allow creating tenant schemas when there is a version - * mismatch. Even more, SchemaIdGetTenantColocationId() would throw an - * error if the underlying pg_dist_schema metadata table has not - * been created yet, which is the case in older versions. For this reason, - * it's safe to assume that it cannot be a tenant schema when there is a - * version mismatch. + * Bail out early if there is a version mismatch between the Citus + * shared library and the installed SQL extension. This prevents + * querying pg_dist_schema when the catalog table may not exist + * (e.g., during multi_extension tests that install versions older + * than 12.0). * - * But it's a bit tricky that we do the same when version checks are - * disabled because then CheckCitusVersion() returns true even if there - * is a version mismatch. And in that case, the tests that are trying to - * create tables (in multi_extension.sql) in older versions would - * fail when deciding whether we should create a tenant table or not. + * When version checks are disabled (citus.enable_version_checks=off), + * CheckCitusVersion() returns true unconditionally, even if there is + * a real version mismatch. In that case, we need to verify that + * pg_dist_schema actually exists before querying it, since the + * multi_extension test installs old versions (e.g. 8.0-1) with + * version checks disabled and creates tables in that state. * - * The downside of doing so is that, for example, we will skip deleting - * the tenant schema entry from pg_dist_schema when dropping a - * tenant schema while the version checks are disabled even if there was - * no version mismatch. But we're okay with that because we don't expect - * users to disable version checks anyway. + * On the normal path (version checks enabled, versions compatible), + * this adds no overhead -- the !EnableVersionChecks branch is skipped + * entirely. */ - if (!EnableVersionChecks || !CheckCitusVersion(DEBUG4)) + if (!CheckCitusVersion(DEBUG4)) { return false; } + if (!EnableVersionChecks) + { + Oid distSchemaOid = get_relname_relid("pg_dist_schema", + PG_CATALOG_NAMESPACE); + if (!OidIsValid(distSchemaOid)) + { + return false; + } + } + return SchemaIdGetTenantColocationId(schemaId) != INVALID_COLOCATION_ID; } From d3330fdfe18272893fbd21e75c011002578e7ba0 Mon Sep 17 00:00:00 2001 From: Muhammad Usama Date: Mon, 2 Mar 2026 16:40:49 +0500 Subject: [PATCH 3/7] Shard move in block_writes mode fails with idle_in_transaction_session_timeout on metadata workers (#8484) ### Description When performing a shard move using block_writes transfer mode (either directly via citus_move_shard_placement or through the background rebalancer), the operation can fail with: ``` ERROR: terminating connection due to idle-in-transaction timeout CONTEXT: while executing command on : ``` The failing worker is a metadata worker that is neither the source nor the target of the shard move. ### Root Cause LockShardListMetadataOnWorkers() opens coordinated transactions on all metadata workers to acquire advisory shard metadata locks via SELECT lock_shard_metadata(...). These transactions remain open until the entire shard move completes and the coordinated transaction commits. In block_writes mode, the data copy phase (CopyShardsToNode) runs synchronously between the source and target workers. Metadata workers not involved in the copy have no commands to execute and their connections sit completely idle-in-transaction for the entire duration of the data copy. For large shards, the copy can take significantly longer than common idle_in_transaction_session_timeout values, When the timeout fires on an uninvolved worker, PostgreSQL terminates the connection, causing the shard move to fail. This also affects shard splits, since they follow the same code path through LockShardListMetadataOnWorkers. ### Fix LockShardListMetadataOnWorkers() should send SET LOCAL idle_in_transaction_session_timeout = 0 on each metadata worker connection before acquiring the locks. SET LOCAL scopes the change to the current transaction only, so normal sessions on the workers are unaffected. --- src/backend/distributed/utils/resource_lock.c | 12 ++- .../shard_move_constraints_blocking.out | 84 +++++++++++++++++++ .../sql/shard_move_constraints_blocking.sql | 38 +++++++++ 3 files changed, 133 insertions(+), 1 deletion(-) diff --git a/src/backend/distributed/utils/resource_lock.c b/src/backend/distributed/utils/resource_lock.c index 1dbc84c42b2..9edfc4943a9 100644 --- a/src/backend/distributed/utils/resource_lock.c +++ b/src/backend/distributed/utils/resource_lock.c @@ -405,7 +405,17 @@ LockShardListMetadataOnWorkers(LOCKMODE lockmode, List *shardIntervalList) appendStringInfo(lockCommand, "])"); - SendCommandToWorkersWithMetadata(lockCommand->data); + /* + * Disable idle_in_transaction_session_timeout on metadata workers before + * acquiring locks. In block_writes mode, these connections stay open for + * the entire shard copy which can take hours for large shards. Without + * this, the timeout would kill the connection and fail the move. + * SET LOCAL scopes the change to this transaction only. + */ + List *commandList = list_make2( + "SET LOCAL idle_in_transaction_session_timeout = 0", + lockCommand->data); + SendCommandListToWorkersWithMetadata(commandList); } diff --git a/src/test/regress/expected/shard_move_constraints_blocking.out b/src/test/regress/expected/shard_move_constraints_blocking.out index 66dec069e7a..61dbf41ec04 100644 --- a/src/test/regress/expected/shard_move_constraints_blocking.out +++ b/src/test/regress/expected/shard_move_constraints_blocking.out @@ -399,3 +399,87 @@ drop cascades to table "blocking shard Move Fkeys Indexes".reference_table drop cascades to table "blocking shard Move Fkeys Indexes".reference_table_8970028 drop cascades to table "blocking shard Move Fkeys Indexes".index_backed_rep_identity DROP ROLE mx_rebalancer_blocking_role_ent; +-- Test: block_writes shard move succeeds even when workers have a low +-- idle_in_transaction_session_timeout. LockShardListMetadataOnWorkers opens +-- coordinated transactions on ALL metadata workers before the data copy. +-- Workers not involved in the copy sit idle-in-transaction for the entire +-- duration. Without the SET LOCAL override, the timeout would kill those +-- connections and fail the move. +SET citus.next_shard_id TO 8980000; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; +CREATE SCHEMA blocking_move_idle_timeout; +SET search_path TO blocking_move_idle_timeout; +-- set a very low idle_in_transaction_session_timeout on all nodes +SELECT 1 FROM run_command_on_all_nodes( + 'ALTER SYSTEM SET idle_in_transaction_session_timeout = ''1s'''); + ?column? +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +SELECT 1 FROM run_command_on_all_nodes('SELECT pg_reload_conf()'); + ?column? +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +-- allow the reload to take effect +SELECT pg_sleep(0.5); + pg_sleep +--------------------------------------------------------------------- + +(1 row) + +CREATE TABLE test_move(id int PRIMARY KEY, val text); +SELECT create_distributed_table('test_move', 'id'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +INSERT INTO test_move SELECT i, 'val_' || i FROM generate_series(1, 100) i; +-- move a shard using block_writes; should succeed despite the 1s timeout +SELECT citus_move_shard_placement(8980000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode:='block_writes'); + citus_move_shard_placement +--------------------------------------------------------------------- + +(1 row) + +SELECT public.wait_for_resource_cleanup(); + wait_for_resource_cleanup +--------------------------------------------------------------------- + +(1 row) + +-- verify data integrity after move +SELECT count(*) FROM test_move; + count +--------------------------------------------------------------------- + 100 +(1 row) + +-- cleanup: restore idle_in_transaction_session_timeout +SELECT 1 FROM run_command_on_all_nodes( + 'ALTER SYSTEM RESET idle_in_transaction_session_timeout'); + ?column? +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +SELECT 1 FROM run_command_on_all_nodes('SELECT pg_reload_conf()'); + ?column? +--------------------------------------------------------------------- + 1 + 1 + 1 +(3 rows) + +DROP SCHEMA blocking_move_idle_timeout CASCADE; +NOTICE: drop cascades to table test_move diff --git a/src/test/regress/sql/shard_move_constraints_blocking.sql b/src/test/regress/sql/shard_move_constraints_blocking.sql index 66b58f42b9c..acbaca76ab9 100644 --- a/src/test/regress/sql/shard_move_constraints_blocking.sql +++ b/src/test/regress/sql/shard_move_constraints_blocking.sql @@ -222,3 +222,41 @@ ALTER TABLE sensors_2020_01_01 DROP CONSTRAINT fkey_from_child_to_child; \c - postgres - :master_port DROP SCHEMA "blocking shard Move Fkeys Indexes" CASCADE; DROP ROLE mx_rebalancer_blocking_role_ent; + +-- Test: block_writes shard move succeeds even when workers have a low +-- idle_in_transaction_session_timeout. LockShardListMetadataOnWorkers opens +-- coordinated transactions on ALL metadata workers before the data copy. +-- Workers not involved in the copy sit idle-in-transaction for the entire +-- duration. Without the SET LOCAL override, the timeout would kill those +-- connections and fail the move. +SET citus.next_shard_id TO 8980000; +SET citus.shard_count TO 4; +SET citus.shard_replication_factor TO 1; + +CREATE SCHEMA blocking_move_idle_timeout; +SET search_path TO blocking_move_idle_timeout; + +-- set a very low idle_in_transaction_session_timeout on all nodes +SELECT 1 FROM run_command_on_all_nodes( + 'ALTER SYSTEM SET idle_in_transaction_session_timeout = ''1s'''); +SELECT 1 FROM run_command_on_all_nodes('SELECT pg_reload_conf()'); +-- allow the reload to take effect +SELECT pg_sleep(0.5); + +CREATE TABLE test_move(id int PRIMARY KEY, val text); +SELECT create_distributed_table('test_move', 'id'); +INSERT INTO test_move SELECT i, 'val_' || i FROM generate_series(1, 100) i; + +-- move a shard using block_writes; should succeed despite the 1s timeout +SELECT citus_move_shard_placement(8980000, 'localhost', :worker_1_port, 'localhost', :worker_2_port, shard_transfer_mode:='block_writes'); +SELECT public.wait_for_resource_cleanup(); + +-- verify data integrity after move +SELECT count(*) FROM test_move; + +-- cleanup: restore idle_in_transaction_session_timeout +SELECT 1 FROM run_command_on_all_nodes( + 'ALTER SYSTEM RESET idle_in_transaction_session_timeout'); +SELECT 1 FROM run_command_on_all_nodes('SELECT pg_reload_conf()'); + +DROP SCHEMA blocking_move_idle_timeout CASCADE; From 366fd644afdc270c9efca7ef36bdeb86225ab91b Mon Sep 17 00:00:00 2001 From: ibrahim halatci Date: Wed, 4 Mar 2026 14:54:38 +0300 Subject: [PATCH 4/7] Fix all open Dependabot Python alerts in Pipfiles (#8488) ## Summary This PR addresses all currently open Dependabot alerts in this repository by updating vulnerable Python dependencies in both mirrored regression-test environments: - src/test/regress/Pipfile / Pipfile.lock - .devcontainer/src/test/regress/Pipfile / Pipfile.lock ## Updated dependencies - cryptography: 44.0.3 -> 46.0.5 (patched: >=46.0.5) - Werkzeug: 3.1.4 -> 3.1.5 (patched: >=3.1.5) - filelock: resolved to 3.25.0 (patched: >=3.20.3) - pyasn1: resolved to 0.6.2 (patched: >=0.6.2) ## Alerts covered Closes Dependabot alerts: #98, #99, #100, #101, #102, #103, #104, #105, #106, #107, #108, #109. ## Notes Lockfiles were regenerated with pipenv lock in both directories to ensure consistent, hashed resolution. --- .devcontainer/Dockerfile | 4 +- .devcontainer/src/test/regress/Pipfile | 4 +- .devcontainer/src/test/regress/Pipfile.lock | 194 +++++++++++--------- .github/workflows/build_and_test.yml | 12 +- src/test/regress/Pipfile | 4 +- src/test/regress/Pipfile.lock | 194 +++++++++++--------- 6 files changed, 218 insertions(+), 194 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5e3efdcb2ab..79b2683a453 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -97,7 +97,7 @@ RUN cp -r .pgenv/src .pgenv/pgsql-* .pgenv/config .pgenv-staging/ RUN rm .pgenv-staging/config/default.conf FROM base AS pg18 -RUN MAKEFLAGS="-j $(nproc)" pgenv build 18.1 +RUN MAKEFLAGS="-j $(nproc)" pgenv build 18.3 RUN rm .pgenv/src/*.tar* RUN make -C .pgenv/src/postgresql-*/ clean RUN make -C .pgenv/src/postgresql-*/src/include install @@ -216,7 +216,7 @@ COPY --chown=citus:citus .psqlrc . RUN sudo chown --from=root:root citus:citus -R ~ # sets default pg version -RUN pgenv switch 18.1 +RUN pgenv switch 18.3 # make connecting to the coordinator easy ENV PGPORT=9700 diff --git a/.devcontainer/src/test/regress/Pipfile b/.devcontainer/src/test/regress/Pipfile index 8ade5491338..06e820beb44 100644 --- a/.devcontainer/src/test/regress/Pipfile +++ b/.devcontainer/src/test/regress/Pipfile @@ -16,7 +16,7 @@ tornado = ">=6.5.1,<6.6.0" zstandard = ">=0.25.0" construct = "*" docopt = "==0.6.2" -cryptography = "==44.0.3" +cryptography = "==46.0.5" pytest = "*" psycopg = "*" filelock = "*" @@ -25,7 +25,7 @@ pytest-timeout = "*" pytest-xdist = "*" pytest-repeat = "*" pyyaml = "*" -werkzeug = "==3.1.4" +werkzeug = "==3.1.5" "typing-extensions" = ">=4.13.2,<5" pyperclip = "==1.9.0" diff --git a/.devcontainer/src/test/regress/Pipfile.lock b/.devcontainer/src/test/regress/Pipfile.lock index 5b662d4343f..8d7fadb409d 100644 --- a/.devcontainer/src/test/regress/Pipfile.lock +++ b/.devcontainer/src/test/regress/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5f734fff88a49010712613a14addae5f38347c691d0b3004b09f59ba5c7cc061" + "sha256": "92e863acf1c77d535dc7da881b4e4cf03bb8fa748769c845aaee27d86d94657c" }, "pipfile-spec": 6, "requires": { @@ -292,11 +292,11 @@ }, "certifi": { "hashes": [ - "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", - "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120" + "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", + "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" ], "markers": "python_version >= '3.7'", - "version": "==2026.1.4" + "version": "==2026.2.25" }, "cffi": { "hashes": [ @@ -407,47 +407,59 @@ }, "cryptography": { "hashes": [ - "sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259", - "sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43", - "sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645", - "sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8", - "sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44", - "sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d", - "sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f", - "sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d", - "sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54", - "sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9", - "sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137", - "sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f", - "sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c", - "sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334", - "sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c", - "sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b", - "sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2", - "sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375", - "sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88", - "sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5", - "sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647", - "sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c", - "sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359", - "sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5", - "sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d", - "sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028", - "sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01", - "sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904", - "sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d", - "sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93", - "sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06", - "sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff", - "sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76", - "sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff", - "sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759", - "sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4", - "sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053" + "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", + "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", + "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9", + "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356", + "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257", + "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", + "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", + "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c", + "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", + "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", + "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", + "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", + "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", + "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", + "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", + "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4", + "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4", + "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", + "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", + "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", + "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678", + "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", + "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", + "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0", + "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", + "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d", + "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", + "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", + "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981", + "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", + "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", + "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2", + "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", + "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663", + "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", + "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", + "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", + "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a", + "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", + "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", + "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a", + "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826", + "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee", + "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", + "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648", + "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da", + "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", + "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", + "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87" ], "index": "pypi", - "markers": "python_version >= '3.7' and python_full_version not in '3.9.0, 3.9.1'", - "version": "==44.0.3" + "markers": "python_version >= '3.8' and python_full_version not in '3.9.0, 3.9.1'", + "version": "==46.0.5" }, "docopt": { "hashes": [ @@ -466,12 +478,12 @@ }, "filelock": { "hashes": [ - "sha256:a2241ff4ddde2a7cebddf78e39832509cb045d18ec1a09d7248d6bfc6bfbbe64", - "sha256:fbba7237d6ea277175a32c54bb71ef814a8546d8601269e1bfc388de333974e8" + "sha256:5ccf8069f7948f494968fc0713c10e5c182a9c9d9eef3a636307a20c2490f047", + "sha256:8f00faf3abf9dc730a1ffe9c354ae5c04e079ab7d3a683b7c32da5dd05f26af3" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==3.20.2" + "version": "==3.25.0" }, "flask": { "hashes": [ @@ -664,24 +676,24 @@ }, "mitmproxy-linux": { "hashes": [ - "sha256:0bea9353c71ebfd2174f6730b3fd0fdff3adea1aa15450035bed3b83e36ef455", - "sha256:2238455e65970382825baed2e998601ea82d8dcaae51bd8ee0859d596524a822", - "sha256:fbcb25316e95d0b2b5ced4e0cc3d90fdb1b7169300a005cc79339894d665363a" + "sha256:94b10fee02aa42287739623cef921e1a53955005d45c9e2fa309ae9f0bf8d37d", + "sha256:b4413e27c692f30036ad6d73432826e728ede026fac8e51651d0c545dd0177f2", + "sha256:ee842865a05f69196004ddcb29d50af0602361d9d6acee04f370f7e01c3674e8" ], "markers": "python_version >= '3.12'", - "version": "==0.12.8" + "version": "==0.12.9" }, "mitmproxy-rs": { "hashes": [ - "sha256:14ea236d0950ab35d667b78b5fe15d43e7345e166e22144624a1283edc78443e", - "sha256:16afd0fc1a00d586ffe2027d217908c3e0389d7d0897eccda6e59fda991e89ba", - "sha256:739591f696cf29913302a72fa9644cf97228774604304a2ea3987fe5588d231c", - "sha256:b0ead519f5a4ab019e7912544c0642f28f8336036ef1480e42a772a8cc947550", - "sha256:c5b0799808a4de0ee60e8f350043820ad56eea738ce3ce25d5c6faaa245b6c9a" + "sha256:1fb9fb4aac9ecb82e2c3c5c439ef5e4961be7934d80ade5e9a99c0a944b8ea2f", + "sha256:1fd716e87da8be3c62daa4325a5ff42bedd951fb8614c5f66caa94b7c21e2593", + "sha256:245922663440330c4b5a36d0194ed559b1dbd5e38545db2eb947180ed12a5e92", + "sha256:afeb3a2da2bc26474e1a2febaea4432430c5fde890dfce33bc4c1e65e6baef1b", + "sha256:c6ffc35c002c675cac534442d92d1cdebd66fafd63754ad33b92ae968ea6e449" ], "index": "pypi", "markers": "python_version >= '3.12'", - "version": "==0.12.8" + "version": "==0.12.9" }, "msgpack": { "hashes": [ @@ -753,11 +765,11 @@ }, "packaging": { "hashes": [ - "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", - "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", + "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" ], "markers": "python_version >= '3.8'", - "version": "==25.0" + "version": "==26.0" }, "pluggy": { "hashes": [ @@ -769,12 +781,12 @@ }, "psycopg": { "hashes": [ - "sha256:3e94bc5f4690247d734599af56e51bae8e0db8e4311ea413f801fef82b14a99b", - "sha256:707a67975ee214d200511177a6a80e56e654754c9afca06a7194ea6bbfde9ca7" + "sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9", + "sha256:f96525a72bcfade6584ab17e89de415ff360748c766f0106959144dcbb38c698" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==3.3.2" + "version": "==3.3.3" }, "publicsuffix2": { "hashes": [ @@ -785,11 +797,11 @@ }, "pyasn1": { "hashes": [ - "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", - "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034" + "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", + "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b" ], "markers": "python_version >= '3.8'", - "version": "==0.6.1" + "version": "==0.6.2" }, "pyasn1-modules": { "hashes": [ @@ -801,11 +813,11 @@ }, "pycparser": { "hashes": [ - "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", - "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934" + "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", + "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" ], - "markers": "python_version >= '3.8'", - "version": "==2.23" + "markers": "python_version >= '3.10'", + "version": "==3.0" }, "pygments": { "hashes": [ @@ -835,11 +847,11 @@ }, "pyopenssl": { "hashes": [ - "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab", - "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b" + "sha256:1fda6fc034d5e3d179d39e59c1895c9faeaf40a79de5fc4cbbfbe0d36f4a77b6", + "sha256:c981cb0a3fd84e8602d7afc209522773b94c1c2446a3c710a75b06fe1beae329" ], "markers": "python_version >= '3.7'", - "version": "==25.1.0" + "version": "==25.3.0" }, "pyparsing": { "hashes": [ @@ -1109,20 +1121,20 @@ }, "wcwidth": { "hashes": [ - "sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605", - "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1" + "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", + "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159" ], - "markers": "python_version >= '3.6'", - "version": "==0.2.14" + "markers": "python_version >= '3.8'", + "version": "==0.6.0" }, "werkzeug": { "hashes": [ - "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905", - "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e" + "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", + "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==3.1.4" + "version": "==3.1.5" }, "wsproto": { "hashes": [ @@ -1305,12 +1317,12 @@ }, "isort": { "hashes": [ - "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", - "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187" + "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d", + "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75" ], "index": "pypi", "markers": "python_full_version >= '3.10.0'", - "version": "==7.0.0" + "version": "==8.0.1" }, "mccabe": { "hashes": [ @@ -1330,27 +1342,27 @@ }, "packaging": { "hashes": [ - "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", - "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", + "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" ], "markers": "python_version >= '3.8'", - "version": "==25.0" + "version": "==26.0" }, "pathspec": { "hashes": [ - "sha256:62f8558917908d237d399b9b338ef455a814801a4688bc41074b25feefd93472", - "sha256:fa32b1eb775ed9ba8d599b22c5f906dc098113989da2c00bf8b210078ca7fb92" + "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", + "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723" ], "markers": "python_version >= '3.9'", - "version": "==1.0.2" + "version": "==1.0.4" }, "platformdirs": { "hashes": [ - "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", - "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31" + "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd", + "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291" ], "markers": "python_version >= '3.10'", - "version": "==4.5.1" + "version": "==4.9.2" }, "pycodestyle": { "hashes": [ diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a5dc4a4b64b..20c13253812 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -31,12 +31,12 @@ jobs: pgupgrade_image_name: "ghcr.io/citusdata/pgupgradetester" style_checker_image_name: "ghcr.io/citusdata/stylechecker" style_checker_tools_version: "0.8.33" - sql_snapshot_pg_version: "18.1" - image_suffix: "-v181ce3c" - pg16_version: '{ "major": "16", "full": "16.11" }' - pg17_version: '{ "major": "17", "full": "17.7" }' - pg18_version: '{ "major": "18", "full": "18.1" }' - upgrade_pg_versions: "16.11-17.7-18.1" + sql_snapshot_pg_version: "18.3" + image_suffix: "-vac4338a" + pg16_version: '{ "major": "16", "full": "16.13" }' + pg17_version: '{ "major": "17", "full": "17.9" }' + pg18_version: '{ "major": "18", "full": "18.3" }' + upgrade_pg_versions: "16.13-17.9-18.3" steps: # Since GHA jobs need at least one step we use a noop step here. - name: Set up parameters diff --git a/src/test/regress/Pipfile b/src/test/regress/Pipfile index 8ade5491338..06e820beb44 100644 --- a/src/test/regress/Pipfile +++ b/src/test/regress/Pipfile @@ -16,7 +16,7 @@ tornado = ">=6.5.1,<6.6.0" zstandard = ">=0.25.0" construct = "*" docopt = "==0.6.2" -cryptography = "==44.0.3" +cryptography = "==46.0.5" pytest = "*" psycopg = "*" filelock = "*" @@ -25,7 +25,7 @@ pytest-timeout = "*" pytest-xdist = "*" pytest-repeat = "*" pyyaml = "*" -werkzeug = "==3.1.4" +werkzeug = "==3.1.5" "typing-extensions" = ">=4.13.2,<5" pyperclip = "==1.9.0" diff --git a/src/test/regress/Pipfile.lock b/src/test/regress/Pipfile.lock index 5b662d4343f..8d7fadb409d 100644 --- a/src/test/regress/Pipfile.lock +++ b/src/test/regress/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5f734fff88a49010712613a14addae5f38347c691d0b3004b09f59ba5c7cc061" + "sha256": "92e863acf1c77d535dc7da881b4e4cf03bb8fa748769c845aaee27d86d94657c" }, "pipfile-spec": 6, "requires": { @@ -292,11 +292,11 @@ }, "certifi": { "hashes": [ - "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", - "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120" + "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", + "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7" ], "markers": "python_version >= '3.7'", - "version": "==2026.1.4" + "version": "==2026.2.25" }, "cffi": { "hashes": [ @@ -407,47 +407,59 @@ }, "cryptography": { "hashes": [ - "sha256:02f55fb4f8b79c1221b0961488eaae21015b69b210e18c386b69de182ebb1259", - "sha256:157f1f3b8d941c2bd8f3ffee0af9b049c9665c39d3da9db2dc338feca5e98a43", - "sha256:192ed30fac1728f7587c6f4613c29c584abdc565d7417c13904708db10206645", - "sha256:21a83f6f35b9cc656d71b5de8d519f566df01e660ac2578805ab245ffd8523f8", - "sha256:25cd194c39fa5a0aa4169125ee27d1172097857b27109a45fadc59653ec06f44", - "sha256:3883076d5c4cc56dbef0b898a74eb6992fdac29a7b9013870b34efe4ddb39a0d", - "sha256:3bb0847e6363c037df8f6ede57d88eaf3410ca2267fb12275370a76f85786a6f", - "sha256:3be3f649d91cb182c3a6bd336de8b61a0a71965bd13d1a04a0e15b39c3d5809d", - "sha256:3f07943aa4d7dad689e3bb1638ddc4944cc5e0921e3c227486daae0e31a05e54", - "sha256:479d92908277bed6e1a1c69b277734a7771c2b78633c224445b5c60a9f4bc1d9", - "sha256:4ffc61e8f3bf5b60346d89cd3d37231019c17a081208dfbbd6e1605ba03fa137", - "sha256:5639c2b16764c6f76eedf722dbad9a0914960d3489c0cc38694ddf9464f1bb2f", - "sha256:58968d331425a6f9eedcee087f77fd3c927c88f55368f43ff7e0a19891f2642c", - "sha256:5d186f32e52e66994dce4f766884bcb9c68b8da62d61d9d215bfe5fb56d21334", - "sha256:5d20cc348cca3a8aa7312f42ab953a56e15323800ca3ab0706b8cd452a3a056c", - "sha256:6866df152b581f9429020320e5eb9794c8780e90f7ccb021940d7f50ee00ae0b", - "sha256:7d5fe7195c27c32a64955740b949070f21cba664604291c298518d2e255931d2", - "sha256:896530bc9107b226f265effa7ef3f21270f18a2026bc09fed1ebd7b66ddf6375", - "sha256:962bc30480a08d133e631e8dfd4783ab71cc9e33d5d7c1e192f0b7c06397bb88", - "sha256:978631ec51a6bbc0b7e58f23b68a8ce9e5f09721940933e9c217068388789fe5", - "sha256:9b4d4a5dbee05a2c390bf212e78b99434efec37b17a4bff42f50285c5c8c9647", - "sha256:ab0b005721cc0039e885ac3503825661bd9810b15d4f374e473f8c89b7d5460c", - "sha256:af653022a0c25ef2e3ffb2c673a50e5a0d02fecc41608f4954176f1933b12359", - "sha256:b0cc66c74c797e1db750aaa842ad5b8b78e14805a9b5d1348dc603612d3e3ff5", - "sha256:b424563394c369a804ecbee9b06dfb34997f19d00b3518e39f83a5642618397d", - "sha256:c138abae3a12a94c75c10499f1cbae81294a6f983b3af066390adee73f433028", - "sha256:c6cd67722619e4d55fdb42ead64ed8843d64638e9c07f4011163e46bc512cf01", - "sha256:c91fc8e8fd78af553f98bc7f2a1d8db977334e4eea302a4bfd75b9461c2d8904", - "sha256:cad399780053fb383dc067475135e41c9fe7d901a97dd5d9c5dfb5611afc0d7d", - "sha256:cb90f60e03d563ca2445099edf605c16ed1d5b15182d21831f58460c48bffb93", - "sha256:dad80b45c22e05b259e33ddd458e9e2ba099c86ccf4e88db7bbab4b747b18d06", - "sha256:dd3db61b8fe5be220eee484a17233287d0be6932d056cf5738225b9c05ef4fff", - "sha256:e28d62e59a4dbd1d22e747f57d4f00c459af22181f0b2f787ea83f5a876d7c76", - "sha256:e909df4053064a97f1e6565153ff8bb389af12c5c8d29c343308760890560aff", - "sha256:f3ffef566ac88f75967d7abd852ed5f182da252d23fac11b4766da3957766759", - "sha256:fc3c9babc1e1faefd62704bb46a69f359a9819eb0292e40df3fb6e3574715cd4", - "sha256:fe19d8bc5536a91a24a8133328880a41831b6c5df54599a8417b62fe015d3053" + "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", + "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", + "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9", + "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356", + "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257", + "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", + "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", + "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c", + "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", + "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", + "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", + "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", + "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", + "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", + "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", + "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4", + "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4", + "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", + "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", + "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", + "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678", + "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", + "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", + "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0", + "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", + "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d", + "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", + "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", + "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981", + "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", + "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", + "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2", + "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", + "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663", + "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", + "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", + "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", + "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a", + "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", + "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", + "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a", + "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826", + "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee", + "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", + "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648", + "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da", + "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", + "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", + "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87" ], "index": "pypi", - "markers": "python_version >= '3.7' and python_full_version not in '3.9.0, 3.9.1'", - "version": "==44.0.3" + "markers": "python_version >= '3.8' and python_full_version not in '3.9.0, 3.9.1'", + "version": "==46.0.5" }, "docopt": { "hashes": [ @@ -466,12 +478,12 @@ }, "filelock": { "hashes": [ - "sha256:a2241ff4ddde2a7cebddf78e39832509cb045d18ec1a09d7248d6bfc6bfbbe64", - "sha256:fbba7237d6ea277175a32c54bb71ef814a8546d8601269e1bfc388de333974e8" + "sha256:5ccf8069f7948f494968fc0713c10e5c182a9c9d9eef3a636307a20c2490f047", + "sha256:8f00faf3abf9dc730a1ffe9c354ae5c04e079ab7d3a683b7c32da5dd05f26af3" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==3.20.2" + "version": "==3.25.0" }, "flask": { "hashes": [ @@ -664,24 +676,24 @@ }, "mitmproxy-linux": { "hashes": [ - "sha256:0bea9353c71ebfd2174f6730b3fd0fdff3adea1aa15450035bed3b83e36ef455", - "sha256:2238455e65970382825baed2e998601ea82d8dcaae51bd8ee0859d596524a822", - "sha256:fbcb25316e95d0b2b5ced4e0cc3d90fdb1b7169300a005cc79339894d665363a" + "sha256:94b10fee02aa42287739623cef921e1a53955005d45c9e2fa309ae9f0bf8d37d", + "sha256:b4413e27c692f30036ad6d73432826e728ede026fac8e51651d0c545dd0177f2", + "sha256:ee842865a05f69196004ddcb29d50af0602361d9d6acee04f370f7e01c3674e8" ], "markers": "python_version >= '3.12'", - "version": "==0.12.8" + "version": "==0.12.9" }, "mitmproxy-rs": { "hashes": [ - "sha256:14ea236d0950ab35d667b78b5fe15d43e7345e166e22144624a1283edc78443e", - "sha256:16afd0fc1a00d586ffe2027d217908c3e0389d7d0897eccda6e59fda991e89ba", - "sha256:739591f696cf29913302a72fa9644cf97228774604304a2ea3987fe5588d231c", - "sha256:b0ead519f5a4ab019e7912544c0642f28f8336036ef1480e42a772a8cc947550", - "sha256:c5b0799808a4de0ee60e8f350043820ad56eea738ce3ce25d5c6faaa245b6c9a" + "sha256:1fb9fb4aac9ecb82e2c3c5c439ef5e4961be7934d80ade5e9a99c0a944b8ea2f", + "sha256:1fd716e87da8be3c62daa4325a5ff42bedd951fb8614c5f66caa94b7c21e2593", + "sha256:245922663440330c4b5a36d0194ed559b1dbd5e38545db2eb947180ed12a5e92", + "sha256:afeb3a2da2bc26474e1a2febaea4432430c5fde890dfce33bc4c1e65e6baef1b", + "sha256:c6ffc35c002c675cac534442d92d1cdebd66fafd63754ad33b92ae968ea6e449" ], "index": "pypi", "markers": "python_version >= '3.12'", - "version": "==0.12.8" + "version": "==0.12.9" }, "msgpack": { "hashes": [ @@ -753,11 +765,11 @@ }, "packaging": { "hashes": [ - "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", - "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", + "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" ], "markers": "python_version >= '3.8'", - "version": "==25.0" + "version": "==26.0" }, "pluggy": { "hashes": [ @@ -769,12 +781,12 @@ }, "psycopg": { "hashes": [ - "sha256:3e94bc5f4690247d734599af56e51bae8e0db8e4311ea413f801fef82b14a99b", - "sha256:707a67975ee214d200511177a6a80e56e654754c9afca06a7194ea6bbfde9ca7" + "sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9", + "sha256:f96525a72bcfade6584ab17e89de415ff360748c766f0106959144dcbb38c698" ], "index": "pypi", "markers": "python_version >= '3.10'", - "version": "==3.3.2" + "version": "==3.3.3" }, "publicsuffix2": { "hashes": [ @@ -785,11 +797,11 @@ }, "pyasn1": { "hashes": [ - "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", - "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034" + "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", + "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b" ], "markers": "python_version >= '3.8'", - "version": "==0.6.1" + "version": "==0.6.2" }, "pyasn1-modules": { "hashes": [ @@ -801,11 +813,11 @@ }, "pycparser": { "hashes": [ - "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", - "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934" + "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", + "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" ], - "markers": "python_version >= '3.8'", - "version": "==2.23" + "markers": "python_version >= '3.10'", + "version": "==3.0" }, "pygments": { "hashes": [ @@ -835,11 +847,11 @@ }, "pyopenssl": { "hashes": [ - "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab", - "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b" + "sha256:1fda6fc034d5e3d179d39e59c1895c9faeaf40a79de5fc4cbbfbe0d36f4a77b6", + "sha256:c981cb0a3fd84e8602d7afc209522773b94c1c2446a3c710a75b06fe1beae329" ], "markers": "python_version >= '3.7'", - "version": "==25.1.0" + "version": "==25.3.0" }, "pyparsing": { "hashes": [ @@ -1109,20 +1121,20 @@ }, "wcwidth": { "hashes": [ - "sha256:4d478375d31bc5395a3c55c40ccdf3354688364cd61c4f6adacaa9215d0b3605", - "sha256:a7bb560c8aee30f9957e5f9895805edd20602f2d7f720186dfd906e82b4982e1" + "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", + "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159" ], - "markers": "python_version >= '3.6'", - "version": "==0.2.14" + "markers": "python_version >= '3.8'", + "version": "==0.6.0" }, "werkzeug": { "hashes": [ - "sha256:2ad50fb9ed09cc3af22c54698351027ace879a0b60a3b5edf5730b2f7d876905", - "sha256:cd3cd98b1b92dc3b7b3995038826c68097dcb16f9baa63abe35f20eafeb9fe5e" + "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", + "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==3.1.4" + "version": "==3.1.5" }, "wsproto": { "hashes": [ @@ -1305,12 +1317,12 @@ }, "isort": { "hashes": [ - "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", - "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187" + "sha256:171ac4ff559cdc060bcfff550bc8404a486fee0caab245679c2abe7cb253c78d", + "sha256:28b89bc70f751b559aeca209e6120393d43fbe2490de0559662be7a9787e3d75" ], "index": "pypi", "markers": "python_full_version >= '3.10.0'", - "version": "==7.0.0" + "version": "==8.0.1" }, "mccabe": { "hashes": [ @@ -1330,27 +1342,27 @@ }, "packaging": { "hashes": [ - "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", - "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", + "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529" ], "markers": "python_version >= '3.8'", - "version": "==25.0" + "version": "==26.0" }, "pathspec": { "hashes": [ - "sha256:62f8558917908d237d399b9b338ef455a814801a4688bc41074b25feefd93472", - "sha256:fa32b1eb775ed9ba8d599b22c5f906dc098113989da2c00bf8b210078ca7fb92" + "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", + "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723" ], "markers": "python_version >= '3.9'", - "version": "==1.0.2" + "version": "==1.0.4" }, "platformdirs": { "hashes": [ - "sha256:61d5cdcc6065745cdd94f0f878977f8de9437be93de97c1c12f853c9c0cdcbda", - "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31" + "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd", + "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291" ], "markers": "python_version >= '3.10'", - "version": "==4.5.1" + "version": "==4.9.2" }, "pycodestyle": { "hashes": [ From ad2140d30aa9bba782e4e592082ef0d398c41ebd Mon Sep 17 00:00:00 2001 From: Vinod Sridharan <14185211+visridha@users.noreply.github.com> Date: Tue, 10 Mar 2026 11:08:07 -0700 Subject: [PATCH 5/7] Support extended stats creation on expressions (#8501) DESCRIPTION: Support extended stats creation on expressions CREATE STATISTICS today gets pushed to the shards only if the statistics is on a column expression. However, PG14 added support for expressions (opExpr, FuncExprs) etc. This change adds support for it under a GUC. This reuses the same logic as CHECK constraints to parse, transform, and deparse the exprs and add it to the statistics call. --- .../deparser/deparse_statistics_stmts.c | 105 ++++++++++++++- src/backend/distributed/shared_library_init.c | 14 ++ src/include/distributed/commands.h | 1 + .../regress/expected/propagate_statistics.out | 122 +++++++++++++----- src/test/regress/sql/propagate_statistics.sql | 21 +++ 5 files changed, 222 insertions(+), 41 deletions(-) diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index f352b83930f..92eeaa08aee 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -15,11 +15,15 @@ #include "catalog/namespace.h" #include "lib/stringinfo.h" #include "nodes/nodes.h" +#include "parser/parse_expr.h" #include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/ruleutils.h" #include "pg_version_constants.h" #include "distributed/citus_ruleutils.h" +#include "distributed/commands.h" #include "distributed/deparser.h" #include "distributed/listutils.h" #include "distributed/relay_utility.h" @@ -35,6 +39,8 @@ static void AppendStatTypes(StringInfo buf, CreateStatsStmt *stmt); static void AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt); static void AppendTableName(StringInfo buf, CreateStatsStmt *stmt); +bool EnableUnsafeStatisticsExpressions = false; + char * DeparseCreateStatisticsStmt(Node *node) { @@ -231,6 +237,42 @@ AppendStatTypes(StringInfo buf, CreateStatsStmt *stmt) } +/* See ruleutils.c in postgres for the logic here. */ +static bool +looks_like_function(Node *node) +{ + if (node == NULL) + { + return false; /* probably shouldn't happen */ + } + switch (nodeTag(node)) + { + case T_FuncExpr: + { + /* OK, unless it's going to deparse as a cast */ + return (((FuncExpr *) node)->funcformat == COERCE_EXPLICIT_CALL || + ((FuncExpr *) node)->funcformat == COERCE_SQL_SYNTAX); + } + + case T_NullIfExpr: + case T_CoalesceExpr: + case T_MinMaxExpr: + case T_SQLValueFunction: + case T_XmlExpr: + { + /* these are all accepted by func_expr_common_subexpr */ + return true; + } + + default: + { + break; + } + } + return false; +} + + static void AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) { @@ -240,15 +282,64 @@ AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) { if (!column->name) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only simple column references are allowed " - "in CREATE STATISTICS"))); + if (EnableUnsafeStatisticsExpressions) + { + /* + * Since these expressions are parser statements, we first call + * transform to get the transformed Expr tree, and then deparse + * the transformed tree. This is similar to the logic found in + * deparse_table_stmts for check constraints. + */ + if (list_length(stmt->relations) != 1) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "cannot use expressions in CREATE STATISTICS with multiple relations"))); + } + + RangeVar *rel = (RangeVar *) linitial(stmt->relations); + bool missingOk = false; + Oid relOid = RangeVarGetRelid(rel, AccessShareLock, missingOk); + + /* Add table name to the name space in parse state. Otherwise column names + * cannot be found. + */ + Relation relation = table_open(relOid, AccessShareLock); + ParseState *pstate = make_parsestate(NULL); + AddRangeTableEntryToQueryCompat(pstate, relation); + Node *exprCooked = transformExpr(pstate, column->expr, + EXPR_KIND_STATS_EXPRESSION); + + char *relationName = get_rel_name(relOid); + List *relationCtx = deparse_context_for(relationName, relOid); + + char *exprSql = deparse_expression(exprCooked, relationCtx, false, false); + relation_close(relation, NoLock); + + /* Need parens if it's not a bare function call */ + if (looks_like_function(exprCooked)) + { + appendStringInfoString(buf, exprSql); + } + else + { + appendStringInfo(buf, "(%s)", exprSql); + } + } + else + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only simple column references are allowed " + "in CREATE STATISTICS"))); + } } + else + { + const char *columnName = quote_identifier(column->name); - const char *columnName = quote_identifier(column->name); - - appendStringInfoString(buf, columnName); + appendStringInfoString(buf, columnName); + } if (column != llast(stmt->exprs)) { diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index d11a4257bc6..5ae0af9428f 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -1605,6 +1605,20 @@ RegisterCitusConfigVariables(void) GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); + DefineCustomBoolVariable( + "citus.enable_unsafe_statistics_expressions", + gettext_noop("Enables the use of expressions in CREATE STATISTICS calls"), + gettext_noop( + "CREATE STATISTICS in citus currently only supports column name references." + "Enabling this GUC allows the use of expressions (introduced in PG14)," + "but the additional constraint validation on the expression to fail invalid expressions" + "is not validated, and so it is advised to use with caution."), + &EnableUnsafeStatisticsExpressions, + false, + PGC_USERSET, + GUC_STANDARD, + NULL, NULL, NULL); + DefineCustomBoolVariable( "citus.enable_unsafe_triggers", gettext_noop("Enables arbitrary triggers on distributed tables which may cause " diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index 2d8ed3b2f09..da3c7ab8b4b 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -37,6 +37,7 @@ extern bool EnableLocalReferenceForeignKeys; extern bool AllowUnsafeConstraints; extern bool EnableUnsafeTriggers; +extern bool EnableUnsafeStatisticsExpressions; extern int MaxMatViewSizeToAutoRecreate; diff --git a/src/test/regress/expected/propagate_statistics.out b/src/test/regress/expected/propagate_statistics.out index 02563c22ae1..208452e1e34 100644 --- a/src/test/regress/expected/propagate_statistics.out +++ b/src/test/regress/expected/propagate_statistics.out @@ -53,6 +53,28 @@ SELECT create_distributed_table ('test_stats3','a'); (1 row) +-- test creating custom stats with expressions and distributing it. +CREATE TABLE sc2.test_stats_expr ( + a int, + b int, + c float8 +); +CREATE STATISTICS s_expr ON (a + b / 2) FROM sc2.test_stats_expr; +-- succeeds since we replicate it into the shards. +SELECT create_distributed_table('sc2.test_stats_expr', 'a'); + create_distributed_table +--------------------------------------------------------------------- + +(1 row) + +-- fails +CREATE STATISTICS s_expr_post ON (a - (b * 2)),round(c) FROM sc2.test_stats_expr; +ERROR: only simple column references are allowed in CREATE STATISTICS +-- succeeds. +set citus.enable_unsafe_statistics_expressions TO on; +-- add another expression stats on the distributed table should work. +CREATE STATISTICS s_expr_post ON (a - (b * 2)), round(c) FROM sc2.test_stats_expr; +reset citus.enable_unsafe_statistics_expressions; -- test dropping statistics CREATE TABLE test_stats4 ( a int, @@ -112,7 +134,7 @@ WHERE stxnamespace IN ( ) AND stxname SIMILAR TO '%\_\d+' ORDER BY stxname ASC; - stxname + stxname --------------------------------------------------------------------- neW'Stat_980096 neW'Stat_980098 @@ -162,22 +184,54 @@ ORDER BY stxname ASC; s2_980058 s2_980060 s2_980062 - s9_980129 - s9_980131 - s9_980133 - s9_980135 - s9_980137 - s9_980139 - s9_980141 - s9_980143 - s9_980145 - s9_980147 - s9_980149 - s9_980151 - s9_980153 - s9_980155 - s9_980157 - s9_980159 + s9_980161 + s9_980163 + s9_980165 + s9_980167 + s9_980169 + s9_980171 + s9_980173 + s9_980175 + s9_980177 + s9_980179 + s9_980181 + s9_980183 + s9_980185 + s9_980187 + s9_980189 + s9_980191 + s_expr_980128 + s_expr_980130 + s_expr_980132 + s_expr_980134 + s_expr_980136 + s_expr_980138 + s_expr_980140 + s_expr_980142 + s_expr_980144 + s_expr_980146 + s_expr_980148 + s_expr_980150 + s_expr_980152 + s_expr_980154 + s_expr_980156 + s_expr_980158 + s_expr_post_980128 + s_expr_post_980130 + s_expr_post_980132 + s_expr_post_980134 + s_expr_post_980136 + s_expr_post_980138 + s_expr_post_980140 + s_expr_post_980142 + s_expr_post_980144 + s_expr_post_980146 + s_expr_post_980148 + s_expr_post_980150 + s_expr_post_980152 + s_expr_post_980154 + s_expr_post_980156 + s_expr_post_980158 st1_new_980064 st1_new_980066 st1_new_980068 @@ -194,23 +248,23 @@ ORDER BY stxname ASC; st1_new_980090 st1_new_980092 st1_new_980094 - stats_xy_980161 - stats_xy_980163 - stats_xy_980165 - stats_xy_980167 - stats_xy_980169 - stats_xy_980171 - stats_xy_980173 - stats_xy_980175 - stats_xy_980177 - stats_xy_980179 - stats_xy_980181 - stats_xy_980183 - stats_xy_980185 - stats_xy_980187 - stats_xy_980189 - stats_xy_980191 -(96 rows) + stats_xy_980193 + stats_xy_980195 + stats_xy_980197 + stats_xy_980199 + stats_xy_980201 + stats_xy_980203 + stats_xy_980205 + stats_xy_980207 + stats_xy_980209 + stats_xy_980211 + stats_xy_980213 + stats_xy_980215 + stats_xy_980217 + stats_xy_980219 + stats_xy_980221 + stats_xy_980223 +(128 rows) SELECT count(DISTINCT stxnamespace) FROM pg_statistic_ext diff --git a/src/test/regress/sql/propagate_statistics.sql b/src/test/regress/sql/propagate_statistics.sql index 7e1f2fa1822..272ff4bd31c 100644 --- a/src/test/regress/sql/propagate_statistics.sql +++ b/src/test/regress/sql/propagate_statistics.sql @@ -44,6 +44,27 @@ CREATE SCHEMA sc2; CREATE STATISTICS sc2."neW'Stat" ON a,b FROM test_stats3; SELECT create_distributed_table ('test_stats3','a'); +-- test creating custom stats with expressions and distributing it. +CREATE TABLE sc2.test_stats_expr ( + a int, + b int, + c float8 +); +CREATE STATISTICS s_expr ON (a + b / 2) FROM sc2.test_stats_expr; + +-- succeeds since we replicate it into the shards. +SELECT create_distributed_table('sc2.test_stats_expr', 'a'); + +-- fails +CREATE STATISTICS s_expr_post ON (a - (b * 2)),round(c) FROM sc2.test_stats_expr; + +-- succeeds. +set citus.enable_unsafe_statistics_expressions TO on; + +-- add another expression stats on the distributed table should work. +CREATE STATISTICS s_expr_post ON (a - (b * 2)), round(c) FROM sc2.test_stats_expr; +reset citus.enable_unsafe_statistics_expressions; + -- test dropping statistics CREATE TABLE test_stats4 ( a int, From 84ddcd98104f22496b1ea97594e198b12e743ee7 Mon Sep 17 00:00:00 2001 From: Onur Tirtir Date: Fri, 13 Mar 2026 11:46:07 +0300 Subject: [PATCH 6/7] Remove citus.enable_unsafe_statistics_expressions GUC and always enable the feature (#8512) Removing the GUC introduced at https://github.com/citusdata/citus/pull/8501 as it's not needed at all. --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: onurctirtir <16804727+onurctirtir@users.noreply.github.com> --- .../deparser/deparse_statistics_stmts.c | 84 ++++++++----------- src/backend/distributed/shared_library_init.c | 14 ---- src/include/distributed/commands.h | 1 - src/test/regress/expected/pg14.out | 4 +- .../regress/expected/propagate_statistics.out | 8 +- src/test/regress/sql/pg14.sql | 3 +- src/test/regress/sql/propagate_statistics.sql | 9 +- 7 files changed, 40 insertions(+), 83 deletions(-) diff --git a/src/backend/distributed/deparser/deparse_statistics_stmts.c b/src/backend/distributed/deparser/deparse_statistics_stmts.c index 92eeaa08aee..085b194c36a 100644 --- a/src/backend/distributed/deparser/deparse_statistics_stmts.c +++ b/src/backend/distributed/deparser/deparse_statistics_stmts.c @@ -39,8 +39,6 @@ static void AppendStatTypes(StringInfo buf, CreateStatsStmt *stmt); static void AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt); static void AppendTableName(StringInfo buf, CreateStatsStmt *stmt); -bool EnableUnsafeStatisticsExpressions = false; - char * DeparseCreateStatisticsStmt(Node *node) { @@ -282,56 +280,46 @@ AppendColumnNames(StringInfo buf, CreateStatsStmt *stmt) { if (!column->name) { - if (EnableUnsafeStatisticsExpressions) + /* + * Since these expressions are parser statements, we first call + * transform to get the transformed Expr tree, and then deparse + * the transformed tree. This is similar to the logic found in + * deparse_table_stmts for check constraints. + */ + if (list_length(stmt->relations) != 1) + { + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "cannot use expressions in CREATE STATISTICS with multiple relations"))); + } + + RangeVar *rel = (RangeVar *) linitial(stmt->relations); + bool missingOk = false; + Oid relOid = RangeVarGetRelid(rel, AccessShareLock, missingOk); + + /* Add table name to the name space in parse state. Otherwise column names + * cannot be found. + */ + Relation relation = table_open(relOid, AccessShareLock); + ParseState *pstate = make_parsestate(NULL); + AddRangeTableEntryToQueryCompat(pstate, relation); + Node *exprCooked = transformExpr(pstate, column->expr, + EXPR_KIND_STATS_EXPRESSION); + + char *relationName = get_rel_name(relOid); + List *relationCtx = deparse_context_for(relationName, relOid); + + char *exprSql = deparse_expression(exprCooked, relationCtx, false, false); + relation_close(relation, NoLock); + + /* Need parens if it's not a bare function call */ + if (looks_like_function(exprCooked)) { - /* - * Since these expressions are parser statements, we first call - * transform to get the transformed Expr tree, and then deparse - * the transformed tree. This is similar to the logic found in - * deparse_table_stmts for check constraints. - */ - if (list_length(stmt->relations) != 1) - { - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg( - "cannot use expressions in CREATE STATISTICS with multiple relations"))); - } - - RangeVar *rel = (RangeVar *) linitial(stmt->relations); - bool missingOk = false; - Oid relOid = RangeVarGetRelid(rel, AccessShareLock, missingOk); - - /* Add table name to the name space in parse state. Otherwise column names - * cannot be found. - */ - Relation relation = table_open(relOid, AccessShareLock); - ParseState *pstate = make_parsestate(NULL); - AddRangeTableEntryToQueryCompat(pstate, relation); - Node *exprCooked = transformExpr(pstate, column->expr, - EXPR_KIND_STATS_EXPRESSION); - - char *relationName = get_rel_name(relOid); - List *relationCtx = deparse_context_for(relationName, relOid); - - char *exprSql = deparse_expression(exprCooked, relationCtx, false, false); - relation_close(relation, NoLock); - - /* Need parens if it's not a bare function call */ - if (looks_like_function(exprCooked)) - { - appendStringInfoString(buf, exprSql); - } - else - { - appendStringInfo(buf, "(%s)", exprSql); - } + appendStringInfoString(buf, exprSql); } else { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only simple column references are allowed " - "in CREATE STATISTICS"))); + appendStringInfo(buf, "(%s)", exprSql); } } else diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index 5ae0af9428f..d11a4257bc6 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -1605,20 +1605,6 @@ RegisterCitusConfigVariables(void) GUC_SUPERUSER_ONLY | GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); - DefineCustomBoolVariable( - "citus.enable_unsafe_statistics_expressions", - gettext_noop("Enables the use of expressions in CREATE STATISTICS calls"), - gettext_noop( - "CREATE STATISTICS in citus currently only supports column name references." - "Enabling this GUC allows the use of expressions (introduced in PG14)," - "but the additional constraint validation on the expression to fail invalid expressions" - "is not validated, and so it is advised to use with caution."), - &EnableUnsafeStatisticsExpressions, - false, - PGC_USERSET, - GUC_STANDARD, - NULL, NULL, NULL); - DefineCustomBoolVariable( "citus.enable_unsafe_triggers", gettext_noop("Enables arbitrary triggers on distributed tables which may cause " diff --git a/src/include/distributed/commands.h b/src/include/distributed/commands.h index da3c7ab8b4b..2d8ed3b2f09 100644 --- a/src/include/distributed/commands.h +++ b/src/include/distributed/commands.h @@ -37,7 +37,6 @@ extern bool EnableLocalReferenceForeignKeys; extern bool AllowUnsafeConstraints; extern bool EnableUnsafeTriggers; -extern bool EnableUnsafeStatisticsExpressions; extern int MaxMatViewSizeToAutoRecreate; diff --git a/src/test/regress/expected/pg14.out b/src/test/regress/expected/pg14.out index d29d1bf5a74..ae7b7b0d721 100644 --- a/src/test/regress/expected/pg14.out +++ b/src/test/regress/expected/pg14.out @@ -208,7 +208,7 @@ DETAIL: on server postgres@localhost:xxxxx connectionId: xxxxxxx reindex(TABLESPACE test_tablespace1) index idx; ERROR: tablespace "test_tablespace1" does not exist reset citus.log_remote_commands; --- CREATE STATISTICS only allow simple column references +-- CREATE STATISTICS allows table references too CREATE TABLE tbl1(a timestamp, b int); SELECT create_distributed_table('tbl1','a'); create_distributed_table @@ -216,11 +216,9 @@ SELECT create_distributed_table('tbl1','a'); (1 row) --- the last one should error out CREATE STATISTICS s1 (dependencies) ON a, b FROM tbl1; CREATE STATISTICS s2 (mcv) ON a, b FROM tbl1; CREATE STATISTICS s3 (ndistinct) ON date_trunc('month', a), date_trunc('day', a) FROM tbl1; -ERROR: only simple column references are allowed in CREATE STATISTICS set citus.log_remote_commands to off; -- error out in case of ALTER TABLE .. DETACH PARTITION .. CONCURRENTLY/FINALIZE -- only if it's a distributed partitioned table diff --git a/src/test/regress/expected/propagate_statistics.out b/src/test/regress/expected/propagate_statistics.out index 208452e1e34..7b82fbc5e0d 100644 --- a/src/test/regress/expected/propagate_statistics.out +++ b/src/test/regress/expected/propagate_statistics.out @@ -67,14 +67,8 @@ SELECT create_distributed_table('sc2.test_stats_expr', 'a'); (1 row) --- fails -CREATE STATISTICS s_expr_post ON (a - (b * 2)),round(c) FROM sc2.test_stats_expr; -ERROR: only simple column references are allowed in CREATE STATISTICS --- succeeds. -set citus.enable_unsafe_statistics_expressions TO on; --- add another expression stats on the distributed table should work. +-- add expression stats on the distributed table should work. CREATE STATISTICS s_expr_post ON (a - (b * 2)), round(c) FROM sc2.test_stats_expr; -reset citus.enable_unsafe_statistics_expressions; -- test dropping statistics CREATE TABLE test_stats4 ( a int, diff --git a/src/test/regress/sql/pg14.sql b/src/test/regress/sql/pg14.sql index c12a8c4fa6a..aa9a30fa257 100644 --- a/src/test/regress/sql/pg14.sql +++ b/src/test/regress/sql/pg14.sql @@ -42,10 +42,9 @@ reindex(verbose, TABLESPACE test_tablespace) index idx ; -- should error saying table space doesn't exist reindex(TABLESPACE test_tablespace1) index idx; reset citus.log_remote_commands; --- CREATE STATISTICS only allow simple column references +-- CREATE STATISTICS allows table references too CREATE TABLE tbl1(a timestamp, b int); SELECT create_distributed_table('tbl1','a'); --- the last one should error out CREATE STATISTICS s1 (dependencies) ON a, b FROM tbl1; CREATE STATISTICS s2 (mcv) ON a, b FROM tbl1; CREATE STATISTICS s3 (ndistinct) ON date_trunc('month', a), date_trunc('day', a) FROM tbl1; diff --git a/src/test/regress/sql/propagate_statistics.sql b/src/test/regress/sql/propagate_statistics.sql index 272ff4bd31c..2e2fecfe7a1 100644 --- a/src/test/regress/sql/propagate_statistics.sql +++ b/src/test/regress/sql/propagate_statistics.sql @@ -55,15 +55,8 @@ CREATE STATISTICS s_expr ON (a + b / 2) FROM sc2.test_stats_expr; -- succeeds since we replicate it into the shards. SELECT create_distributed_table('sc2.test_stats_expr', 'a'); --- fails -CREATE STATISTICS s_expr_post ON (a - (b * 2)),round(c) FROM sc2.test_stats_expr; - --- succeeds. -set citus.enable_unsafe_statistics_expressions TO on; - --- add another expression stats on the distributed table should work. +-- add expression stats on the distributed table should work. CREATE STATISTICS s_expr_post ON (a - (b * 2)), round(c) FROM sc2.test_stats_expr; -reset citus.enable_unsafe_statistics_expressions; -- test dropping statistics CREATE TABLE test_stats4 ( From 347d7236231e85c3ba76919dfd9140068121ed7f Mon Sep 17 00:00:00 2001 From: Vinod Sridharan <14185211+visridha@users.noreply.github.com> Date: Tue, 17 Mar 2026 01:35:29 -0700 Subject: [PATCH 7/7] Add support for aggregates with an internal stype (#8505) DESCRIPTION: Add support for aggregates with an internal stype Citus has historically required custom aggregates to not have an internal stype except for specific internal aggregates. This has led to a number of workarounds to get performance and custom aggregates working with distributed tables. This change removes that restriction by mirroring Postgres's use of the SERIALFUNC and DESERIALFUNC to roundtrip state for aggregates' internal stype metadata between workers and coordinators allowing more natural use of custom aggregates in Citus. --- .../planner/multi_logical_optimizer.c | 33 ++++- src/backend/distributed/shared_library_init.c | 16 +++ .../distributed/utils/aggregate_utils.c | 125 ++++++++++++++++-- .../distributed/query_pushdown_planning.h | 1 + .../regress/expected/aggregate_support.out | 73 ++++++++++ src/test/regress/sql/aggregate_support.sql | 39 ++++++ 6 files changed, 274 insertions(+), 13 deletions(-) diff --git a/src/backend/distributed/planner/multi_logical_optimizer.c b/src/backend/distributed/planner/multi_logical_optimizer.c index 6a25005a111..f9b2574ad6e 100644 --- a/src/backend/distributed/planner/multi_logical_optimizer.c +++ b/src/backend/distributed/planner/multi_logical_optimizer.c @@ -64,6 +64,7 @@ int LimitClauseRowFetchCount = -1; /* number of rows to fetch from each task */ double CountDistinctErrorRate = 0.0; /* precision of count(distinct) approximate */ int CoordinatorAggregationStrategy = COORDINATOR_AGGREGATION_ROW_GATHER; +bool AllowAggregateWorkerCombineOnInternalTypes = true; /* Constant used throughout file */ static const uint32 masterTableId = 1; /* first range table reference on the master node */ @@ -281,7 +282,7 @@ static Oid CitusFunctionOidWithSignature(char *functionName, int numargs, Oid *a static Oid WorkerPartialAggOid(void); static Oid WorkerBinaryPartialAggOid(void); static Oid CoordBinaryCombineAggOid(void); -static bool IsTypeBinarySerializable(Oid transitionType); +static bool IsAggTransTypeBinarySerializable(Form_pg_aggregate aggForm); static Oid CoordCombineAggOid(void); static Oid AggregateFunctionOid(const char *functionName, Oid inputType); static Oid TypeOid(Oid schemaId, const char *typeName); @@ -2131,7 +2132,7 @@ MasterAggregateExpression(Aggref *originalAggregate, aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple); combine = aggform->aggcombinefn; useBinaryCoordinatorCombine = aggform->aggtranstype != InvalidOid && - IsTypeBinarySerializable(aggform->aggtranstype); + IsAggTransTypeBinarySerializable(aggform); ReleaseSysCache(aggTuple); } @@ -3296,7 +3297,7 @@ WorkerAggregateExpressionList(Aggref *originalAggregate, aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple); combine = aggform->aggcombinefn; useBinaryWorkerAggregate = (OidIsValid(aggform->aggtranstype) && - IsTypeBinarySerializable(aggform->aggtranstype)); + IsAggTransTypeBinarySerializable(aggform)); ReleaseSysCache(aggTuple); } @@ -3568,6 +3569,18 @@ AggregateEnabledCustom(Aggref *aggregateExpression) bool supportsSafeCombine = typeform->typtype != TYPTYPE_PSEUDO; + if (AllowAggregateWorkerCombineOnInternalTypes && + typeform->oid == INTERNALOID && !supportsSafeCombine) + { + /* check if the type supports a SERIALFUNC/DESERIALFUNC - if it does + * then we can leverage that for safe transfer of the state across the wire. + */ + if (aggform->aggserialfn != InvalidOid && aggform->aggdeserialfn != InvalidOid) + { + supportsSafeCombine = true; + } + } + ReleaseSysCache(aggTuple); ReleaseSysCache(typeTuple); @@ -3848,8 +3861,20 @@ TypeOid(Oid schemaId, const char *typeName) static bool -IsTypeBinarySerializable(Oid transitionType) +IsAggTransTypeBinarySerializable(Form_pg_aggregate aggForm) { + Oid transitionType = aggForm->aggtranstype; + + if (AllowAggregateWorkerCombineOnInternalTypes && + transitionType == INTERNALOID) + { + /* For aggregates with internal transition types, we apply the binary serialization + * check on the output value of the SERIALFUNC. If a serialfunc exists, Postgres + * requires that the serialfunc return a bytea - which will be binary serializable + */ + return (aggForm->aggserialfn != InvalidOid); + } + HeapTuple typeTuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(transitionType)); if (!HeapTupleIsValid(typeTuple)) { diff --git a/src/backend/distributed/shared_library_init.c b/src/backend/distributed/shared_library_init.c index d11a4257bc6..f49cf17050e 100644 --- a/src/backend/distributed/shared_library_init.c +++ b/src/backend/distributed/shared_library_init.c @@ -984,6 +984,22 @@ RegisterCitusConfigVariables(void) GUC_STANDARD, NULL, NULL, NULL); + DefineCustomBoolVariable( + "citus.allow_aggregate_worker_combine_on_internal_types", + gettext_noop("Enables aggregate worker partial aggregates on aggregates that " + "have internal type for the aggregate partial state storage."), + gettext_noop( + "This setting allows the use of pushdown of custom aggregates that have " + "an STYPE that is internal. This is typically okay to do, but if a custom aggregate " + "persists OID information or any node specific data into the state, this can cause " + "weirdness when combining in the coordinator, so this is left as an option to turn off " + "in those cases worker combine functions on internal types."), + &AllowAggregateWorkerCombineOnInternalTypes, + true, + PGC_USERSET, + GUC_STANDARD, + NULL, NULL, NULL); + DefineCustomBoolVariable( "citus.allow_modifications_from_workers_to_replicated_tables", gettext_noop("Enables modifications from workers to replicated " diff --git a/src/backend/distributed/utils/aggregate_utils.c b/src/backend/distributed/utils/aggregate_utils.c index e6354e4875a..df85bf9da6d 100644 --- a/src/backend/distributed/utils/aggregate_utils.c +++ b/src/backend/distributed/utils/aggregate_utils.c @@ -623,7 +623,8 @@ GetAggregateTransitionType(StypeBox *box) "worker_partial_agg_ffunc expects an aggregate with COMBINEFUNC"))); } - if (aggform->aggtranstype == INTERNALOID) + if (aggform->aggtranstype == INTERNALOID && + aggform->aggserialfn == InvalidOid) { ereport(ERROR, (errmsg( @@ -660,6 +661,40 @@ WorkerPartialAggregateApplyFFunc(PG_FUNCTION_ARGS) } +/* If the transtype is internal, we need to use the SERIALFUNC of the aggregate + * to serialize the value in the worker. The COMBINEFUNC on the coordinator will + * then use the DESERIALFUNC to deserialize the value. + */ +static Datum +CheckAndCallSerialFunc(PG_FUNCTION_ARGS, StypeBox *box, bool *outputIsNull) +{ + LOCAL_FCINFO(serialFcInfo, 1); + FmgrInfo serialInfo; + Form_pg_aggregate aggform; + HeapTuple aggtuple = GetAggregateForm(box->agg, &aggform); + + if (aggform->aggserialfn == InvalidOid) + { + ereport(ERROR, + (errmsg( + "worker_partial_agg_ffunc expects aggregates with internal transition state to have a SERIALFUNC"))); + } + + /* Otherwise, first invoke the serialfunc to ensure that we produce a serialiable value from the boxValue */ + Assert(get_func_rettype(aggform->aggserialfn) == BYTEAOID); + fmgr_info(aggform->aggserialfn, &serialInfo); + + InitFunctionCallInfoData(*serialFcInfo, &serialInfo, 1, fcinfo->fncollation, + fcinfo->context, fcinfo->resultinfo); + fcSetArgExt(serialFcInfo, 0, box->value, box->valueNull); + + Datum result = FunctionCallInvoke(serialFcInfo); + *outputIsNull = serialFcInfo->isnull; + ReleaseSysCache(aggtuple); + return result; +} + + /* * worker_partial_agg_ffunc serializes transition state, * essentially implementing the following pseudocode: @@ -682,12 +717,21 @@ worker_partial_agg_ffunc(PG_FUNCTION_ARGS) } Oid transtype = GetAggregateTransitionType(box); + + Datum boxValue = box->value; + bool boxValueNull = box->valueNull; + if (transtype == INTERNALOID) + { + ereport(ERROR, (errmsg("worker_partial_agg_ffunc does not support output" + " of aggregates with INTERNAL transition state"))); + } + getTypeOutputInfo(transtype, &typoutput, &typIsVarlena); fmgr_info(typoutput, &info); InitFunctionCallInfoData(*innerFcinfo, &info, 1, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); - fcSetArgExt(innerFcinfo, 0, box->value, box->valueNull); + fcSetArgExt(innerFcinfo, 0, boxValue, boxValueNull); Datum result = FunctionCallInvoke(innerFcinfo); @@ -721,12 +765,24 @@ worker_binary_partial_agg_ffunc(PG_FUNCTION_ARGS) } Oid transtype = GetAggregateTransitionType(box); + + + Datum boxValue = box->value; + bool boxValueNull = box->valueNull; + if (transtype == INTERNALOID) + { + /* Call and store the output of the SERIALFUNC - the output type + * then is always BYTEAOID. */ + boxValue = CheckAndCallSerialFunc(fcinfo, box, &boxValueNull); + transtype = BYTEAOID; + } + getTypeBinaryOutputInfo(transtype, &typoutput, &typIsVarlena); fmgr_info(typoutput, &info); InitFunctionCallInfoData(*innerFcinfo, &info, 1, fcinfo->fncollation, fcinfo->context, fcinfo->resultinfo); - fcSetArgExt(innerFcinfo, 0, box->value, box->valueNull); + fcSetArgExt(innerFcinfo, 0, boxValue, boxValueNull); Datum result = FunctionCallInvoke(innerFcinfo); @@ -738,6 +794,29 @@ worker_binary_partial_agg_ffunc(PG_FUNCTION_ARGS) } +static Datum +DeserializeBoxValue(Oid deserialFunc, Datum value, bool valueNull, + FunctionCallInfo fcinfo, + bool *outputIsNull) +{ + LOCAL_FCINFO(deserialFcInfo, 3); + FmgrInfo deserialInfo; + + fmgr_info(deserialFunc, &deserialInfo); + + InitFunctionCallInfoData(*deserialFcInfo, &deserialInfo, 2, fcinfo->fncollation, + fcinfo->context, fcinfo->resultinfo); + fcSetArgExt(deserialFcInfo, 0, value, valueNull); + + /* Arg1 is not used and is internal */ + fcSetArgExt(deserialFcInfo, 1, (Datum) 0, false); + + Datum result = FunctionCallInvoke(deserialFcInfo); + *outputIsNull = deserialFcInfo->isnull; + return result; +} + + static Datum CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) { @@ -745,6 +824,7 @@ CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) FmgrInfo info; Form_pg_aggregate aggform; Form_pg_type transtypeform; + Oid deserialFunc = InvalidOid; Datum value; StypeBox *box = NULL; @@ -769,9 +849,16 @@ CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) if (aggform->aggtranstype == INTERNALOID) { - ereport(ERROR, - (errmsg( - "coord_combine_agg_sfunc does not support aggregates with INTERNAL transition state"))); + if (aggform->aggdeserialfn == InvalidOid) + { + ereport(ERROR, + (errmsg( + "coord_combine_agg_sfunc does not support aggregates with INTERNAL transition state"))); + } + else + { + deserialFunc = aggform->aggdeserialfn; + } } Oid combine = aggform->aggcombinefn; @@ -791,10 +878,21 @@ CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) } bool valueNull = PG_ARGISNULL(2); - HeapTuple transtypetuple = GetTypeForm(box->transtype, &transtypeform); - Oid ioparam = getTypeIOParam(transtypetuple); + + /* If the stype is internal, this needs to through first + * deserializing the wire output to the intermediate state. + */ + Oid deserializationType = box->transtype; + if (box->transtype == INTERNALOID) + { + /* For a deserialfunc, the input is a BYTEAOID */ + deserializationType = BYTEAOID; + } + + HeapTuple deserializationTypeTuple = GetTypeForm(deserializationType, &transtypeform); + Oid ioparam = getTypeIOParam(deserializationTypeTuple); Oid deserial = isBinaryInput ? transtypeform->typreceive : transtypeform->typinput; - ReleaseSysCache(transtypetuple); + ReleaseSysCache(deserializationTypeTuple); fmgr_info(deserial, &info); if (valueNull && info.fn_strict) @@ -864,6 +962,15 @@ CoordinatorCombineAggSfuncCore(PG_FUNCTION_ARGS, bool isBinaryInput) valueNull = innerFcinfo->isnull; } + /* If the stype is internal, we need to go through one additional step + * of now calling the deserialfunc to go from the serialized type to + * internal before we call the combine function. + */ + if (box->transtype == INTERNALOID && !valueNull) + { + value = DeserializeBoxValue(deserialFunc, value, valueNull, fcinfo, &valueNull); + } + fmgr_info(combine, &info); if (info.fn_strict) diff --git a/src/include/distributed/query_pushdown_planning.h b/src/include/distributed/query_pushdown_planning.h index 87ff07aeb21..0b69d36c75f 100644 --- a/src/include/distributed/query_pushdown_planning.h +++ b/src/include/distributed/query_pushdown_planning.h @@ -22,6 +22,7 @@ /* Config variables managed via guc.c */ extern bool SubqueryPushdown; extern int ValuesMaterializationThreshold; +extern bool AllowAggregateWorkerCombineOnInternalTypes; extern bool CanPushdownSubquery(Query *subqueryTree, bool outerMostQueryHasLimit); diff --git a/src/test/regress/expected/aggregate_support.out b/src/test/regress/expected/aggregate_support.out index 78cd54292df..7f0b8c4639c 100644 --- a/src/test/regress/expected/aggregate_support.out +++ b/src/test/regress/expected/aggregate_support.out @@ -294,6 +294,79 @@ EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF, BUFFERS OFF) SELECT key -> Seq Scan on aggdata_83674000 aggdata (actual rows=4 loops=1) (17 rows) +-- aggregates with internal stype works. +CREATE AGGREGATE internalsum(int8) ( + sfunc = int8_avg_accum, + stype = internal, + finalfunc = numeric_poly_sum, + combinefunc = int8_avg_combine, + serialfunc = int8_avg_serialize, + deserialfunc = int8_avg_deserialize +); +CREATE AGGREGATE internalsum_noserial(int8) ( + sfunc = int8_avg_accum, + stype = internal, + finalfunc = numeric_poly_sum, + combinefunc = int8_avg_combine +); +SELECT key, internalsum(val), sum(val) from aggdata group by key order by key; + key | internalsum | sum +--------------------------------------------------------------------- + 1 | 2 | 2 + 2 | 10 | 10 + 3 | 4 | 4 + 5 | | + 6 | | + 7 | 8 | 8 + 9 | 0 | 0 +(7 rows) + +-- see that the explain is pushed to the shards. +EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF, BUFFERS OFF) SELECT key, internalsum(val) from aggdata group by key order by key; + QUERY PLAN +--------------------------------------------------------------------- + Sort (actual rows=7 loops=1) + Sort Key: remote_scan.key + Sort Method: quicksort Memory: 25kB + -> HashAggregate (actual rows=7 loops=1) + Group Key: remote_scan.key + -> Custom Scan (Citus Adaptive) (actual rows=10 loops=1) + Task Count: 4 + Tuple data received from nodes: 290 bytes + Tasks Shown: One of 4 + -> Task + Tuple data received from node: 118 bytes + Node: host=localhost port=xxxxx dbname=regression + -> HashAggregate (actual rows=4 loops=1) + Group Key: key + -> Seq Scan on aggdata_83674000 aggdata (actual rows=4 loops=1) +(17 rows) + +-- without a serialfunc always fails. +SELECT key, internalsum_noserial(val), sum(val) from aggdata group by key order by key; +ERROR: unsupported aggregate function internalsum_noserial +-- but this works if we allow coordinator combine +set citus.coordinator_aggregation_strategy to 'row-gather'; +SELECT key, internalsum_noserial(val), sum(val) from aggdata group by key order by key; + key | internalsum_noserial | sum +--------------------------------------------------------------------- + 1 | 2 | 2 + 2 | 10 | 10 + 3 | 4 | 4 + 5 | | + 6 | | + 7 | 8 | 8 + 9 | 0 | 0 +(7 rows) + +set citus.coordinator_aggregation_strategy to 'disabled'; +-- if the GUC is unset, do not allow pushdown. +set citus.allow_aggregate_worker_combine_on_internal_types to off; +SELECT key, internalsum(val), sum(val) from aggdata group by key order by key; +ERROR: unsupported aggregate function internalsum +reset citus.allow_aggregate_worker_combine_on_internal_types; +DROP AGGREGATE internalsum(int8); +DROP AGGREGATE internalsum_noserial(int8); -- binary string aggregation create function binstragg_sfunc(s text, e1 text, e2 text) returns text immutable language plpgsql as $$ diff --git a/src/test/regress/sql/aggregate_support.sql b/src/test/regress/sql/aggregate_support.sql index 3962a83b1fa..7482c37efda 100644 --- a/src/test/regress/sql/aggregate_support.sql +++ b/src/test/regress/sql/aggregate_support.sql @@ -144,6 +144,45 @@ select covar_pop(valf,val)::numeric(10,5), covar_samp(valf,val)::numeric(10,5) f set citus.explain_analyze_sort_method to 'taskId'; EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF, BUFFERS OFF) SELECT key, sum2_strict(val) from aggdata group by key order by key; + +-- aggregates with internal stype works. +CREATE AGGREGATE internalsum(int8) ( + sfunc = int8_avg_accum, + stype = internal, + finalfunc = numeric_poly_sum, + combinefunc = int8_avg_combine, + serialfunc = int8_avg_serialize, + deserialfunc = int8_avg_deserialize +); + +CREATE AGGREGATE internalsum_noserial(int8) ( + sfunc = int8_avg_accum, + stype = internal, + finalfunc = numeric_poly_sum, + combinefunc = int8_avg_combine +); + +SELECT key, internalsum(val), sum(val) from aggdata group by key order by key; + +-- see that the explain is pushed to the shards. +EXPLAIN (ANALYZE ON, COSTS OFF, SUMMARY OFF, TIMING OFF, BUFFERS OFF) SELECT key, internalsum(val) from aggdata group by key order by key; + +-- without a serialfunc always fails. +SELECT key, internalsum_noserial(val), sum(val) from aggdata group by key order by key; + +-- but this works if we allow coordinator combine +set citus.coordinator_aggregation_strategy to 'row-gather'; +SELECT key, internalsum_noserial(val), sum(val) from aggdata group by key order by key; +set citus.coordinator_aggregation_strategy to 'disabled'; + +-- if the GUC is unset, do not allow pushdown. +set citus.allow_aggregate_worker_combine_on_internal_types to off; +SELECT key, internalsum(val), sum(val) from aggdata group by key order by key; +reset citus.allow_aggregate_worker_combine_on_internal_types; + +DROP AGGREGATE internalsum(int8); +DROP AGGREGATE internalsum_noserial(int8); + -- binary string aggregation create function binstragg_sfunc(s text, e1 text, e2 text) returns text immutable language plpgsql as $$