From 1a0dbb1a1a1b63efbd0d4ea1f2a8307c6eec9a36 Mon Sep 17 00:00:00 2001 From: Akshat Nehra Date: Fri, 12 Jun 2026 02:32:27 +0000 Subject: [PATCH] MDEV-39918 Fix crash in degenerate jtbm semi-join with recursive CTE/UNION engine execute_degenerate_jtbm_semi_join() assumes the subquery engine is SINGLE_SELECT_ENGINE and casts to subselect_single_select_engine*. However, recursive CTEs, UNIONs, EXCEPTs, and INTERSECTs use UNION_ENGINE even when the first SELECT has no tables (table_count==0). This causes an invalid downcast and crash. Fix: check that the engine type is SINGLE_SELECT_ENGINE before entering the degenerate path. Non-SINGLE_SELECT_ENGINE subqueries fall through to the normal JTBM materialization path. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc. --- mysql-test/main/mdev_39918.result | 60 +++++++++++++++++++++++++++++++ mysql-test/main/mdev_39918.test | 56 +++++++++++++++++++++++++++++ sql/opt_subselect.cc | 20 +++++++++-- 3 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 mysql-test/main/mdev_39918.result create mode 100644 mysql-test/main/mdev_39918.test diff --git a/mysql-test/main/mdev_39918.result b/mysql-test/main/mdev_39918.result new file mode 100644 index 0000000000000..f330ea9f837c4 --- /dev/null +++ b/mysql-test/main/mdev_39918.result @@ -0,0 +1,60 @@ +# +# MDEV-39918: Crash in execute_degenerate_jtbm_semi_join with non-SINGLE_SELECT_ENGINE +# +# Recursive CTE with window function (original crash) +SELECT 1 FROM (SELECT 1 AS x) AS t +WHERE x IN ( +WITH RECURSIVE x(x) AS (SELECT 10 AS x) +SELECT ROW_NUMBER() OVER (ORDER BY AVG(x)) +FROM (SELECT 1 AS x WHERE 1=0) AS d WHERE x +); +1 +1 +# Truly recursive CTE nested in derived table +SELECT 1 FROM (SELECT 1 AS x) AS outer_t +WHERE x IN ( +SELECT ROW_NUMBER() OVER (ORDER BY AVG(x)) +FROM ( +WITH RECURSIVE x(x) AS (SELECT 1 UNION SELECT x + 1 FROM x WHERE x < 3) +SELECT * FROM x WHERE 1=0 +) AS d WHERE x +); +1 +1 +# Simple UNION subquery (no tables in first SELECT) +SELECT 1 FROM (SELECT 1 AS x) AS t +WHERE x IN (SELECT 1 UNION SELECT 2); +1 +1 +# UNION ALL with degenerate first branch +SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 2) AS t +WHERE x IN (SELECT 1 UNION ALL SELECT 2); +x +1 +2 +# EXCEPT variant +SELECT 1 FROM (SELECT 1 AS x) AS t +WHERE x IN (SELECT 1 EXCEPT SELECT 2); +1 +1 +# INTERSECT variant +SELECT 1 FROM (SELECT 1 AS x) AS t +WHERE x IN (SELECT 1 INTERSECT SELECT 1); +1 +1 +# Non-degenerate subquery (has tables, should work as before) +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1), (2), (3); +SELECT * FROM t1 WHERE a IN (SELECT a FROM t1 WHERE a > 1); +a +2 +3 +DROP TABLE t1; +# UNION with table in one branch, no table in another +CREATE TABLE t2 (a INT); +INSERT INTO t2 VALUES (5); +SELECT * FROM (SELECT 1 AS x) AS t +WHERE x IN (SELECT 1 UNION SELECT a FROM t2); +x +1 +DROP TABLE t2; diff --git a/mysql-test/main/mdev_39918.test b/mysql-test/main/mdev_39918.test new file mode 100644 index 0000000000000..cc9d8f4d37fe1 --- /dev/null +++ b/mysql-test/main/mdev_39918.test @@ -0,0 +1,56 @@ +# +# MDEV-39918: MariaDB crash triggered by recursive CTE with window function +# in IN subquery (degenerate jtbm semi-join) +# + +--echo # +--echo # MDEV-39918: Crash in execute_degenerate_jtbm_semi_join with non-SINGLE_SELECT_ENGINE +--echo # + +--echo # Recursive CTE with window function (original crash) +SELECT 1 FROM (SELECT 1 AS x) AS t +WHERE x IN ( + WITH RECURSIVE x(x) AS (SELECT 10 AS x) + SELECT ROW_NUMBER() OVER (ORDER BY AVG(x)) + FROM (SELECT 1 AS x WHERE 1=0) AS d WHERE x +); + +--echo # Truly recursive CTE nested in derived table +SELECT 1 FROM (SELECT 1 AS x) AS outer_t +WHERE x IN ( + SELECT ROW_NUMBER() OVER (ORDER BY AVG(x)) + FROM ( + WITH RECURSIVE x(x) AS (SELECT 1 UNION SELECT x + 1 FROM x WHERE x < 3) + SELECT * FROM x WHERE 1=0 + ) AS d WHERE x +); + +--echo # Simple UNION subquery (no tables in first SELECT) +SELECT 1 FROM (SELECT 1 AS x) AS t +WHERE x IN (SELECT 1 UNION SELECT 2); + +--echo # UNION ALL with degenerate first branch +SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 2) AS t +WHERE x IN (SELECT 1 UNION ALL SELECT 2); + +--echo # EXCEPT variant +SELECT 1 FROM (SELECT 1 AS x) AS t +WHERE x IN (SELECT 1 EXCEPT SELECT 2); + +--echo # INTERSECT variant +SELECT 1 FROM (SELECT 1 AS x) AS t +WHERE x IN (SELECT 1 INTERSECT SELECT 1); + +--echo # Non-degenerate subquery (has tables, should work as before) +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1), (2), (3); +SELECT * FROM t1 WHERE a IN (SELECT a FROM t1 WHERE a > 1); +DROP TABLE t1; + +--echo # UNION with table in one branch, no table in another +CREATE TABLE t2 (a INT); +INSERT INTO t2 VALUES (5); +SELECT * FROM (SELECT 1 AS x) AS t +WHERE x IN (SELECT 1 UNION SELECT a FROM t2); +DROP TABLE t2; + diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index bbc9434c4cf02..78e0e05072746 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -6552,7 +6552,15 @@ bool setup_degenerate_jtbm_semi_joins(JOIN *join, { JOIN *subq_join= subq_pred->unit->first_select()->join; - if (!subq_join->tables_list || !subq_join->table_count) + /* + The degenerate path requires SINGLE_SELECT_ENGINE. + Other engine types (UNION_ENGINE for UNION/EXCEPT/INTERSECT/ + recursive CTEs, etc.) must go through normal JTBM + materialization instead. + */ + if ((!subq_join->tables_list || !subq_join->table_count) && + subq_pred->engine->engine_type() == + subselect_engine::SINGLE_SELECT_ENGINE) { if (execute_degenerate_jtbm_semi_join(thd, table, @@ -6645,7 +6653,15 @@ bool setup_jtbm_semi_joins(JOIN *join, List *join_list, subq_pred->jtbm_record_count=rows; JOIN *subq_join= subq_pred->unit->first_select()->join; - if (!subq_join->tables_list || !subq_join->table_count) + /* + The degenerate path requires SINGLE_SELECT_ENGINE. + Other engine types (UNION_ENGINE for UNION/EXCEPT/INTERSECT/ + recursive CTEs, etc.) must go through normal JTBM + materialization instead. + */ + if ((!subq_join->tables_list || !subq_join->table_count) && + subq_pred->engine->engine_type() == + subselect_engine::SINGLE_SELECT_ENGINE) { if (!join->is_orig_degenerated && execute_degenerate_jtbm_semi_join(thd, table, subq_pred,