From 538868c59bd06fc5d880ab6ca7f95d149861073d Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 21 May 2026 11:12:31 +0200 Subject: [PATCH 1/2] Cherry-pick of https://github.com/Altinity/ClickHouse/pull/1802 with unresolved conflict markers (resolution in next commit) --- Original cherry-pick message follows: Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272 Antalya 26.3: Support Nullable(Tuple) for Arrow, ArrowStream, ORC, legacy Parquet formats # Conflicts: # src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp # src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp # tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference # tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql --- .../Formats/Impl/CHColumnToArrowColumn.cpp | 13 ++ .../Formats/Impl/MsgPackRowInputFormat.cpp | 18 ++ ...nside_nullable_parquet_roundtrip.reference | 207 ++++++++++++++++++ ...uple_inside_nullable_parquet_roundtrip.sql | 187 ++++++++++++++++ 4 files changed, 425 insertions(+) diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 5a9efde03f5d..26bb56074795 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -575,9 +575,14 @@ namespace DB /// Do not propagate the struct-level null_bytemap to child fields. /// In Arrow, struct-level nulls and child-level nulls are independent; /// child values at null struct positions are undefined. +<<<<<<< HEAD auto name = column_name + "." + nested_names[i]; std::shared_ptr nested_arrow_array = fillArrowArray( name, +======= + fillArrowArray( + column_name + "." + nested_names[i], +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) nested_column, nested_types[i], nullptr, builder.field_builder(static_cast(i)), format_name, @@ -588,8 +593,16 @@ namespace DB children.push_back(nested_arrow_array); } +<<<<<<< HEAD auto null_bitmap = nullBytemapToArrowBitmap(null_bytemap, column_name, format_name, start, end); return checkResult(arrow::StructArray::Make(children, builder.type()->fields(), null_bitmap), column_name, format_name); +======= + for (size_t i = start; i != end; ++i) + { + auto status = (null_bytemap && (*null_bytemap)[i]) ? builder.AppendNull() : builder.Append(); + checkStatus(status, column->getName(), format_name); + } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) } template diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index f8e80355f77c..40803f990a59 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -401,11 +401,27 @@ bool MsgPackVisitor::start_array(size_t size) // NOLINT } else if (isTuple(removeNullable(info_stack.top().type))) { +<<<<<<< HEAD +======= + /// If the type is Nullable, reaching start_array means the value + /// is non-null (for nulls, the parser calls visit_nil instead). + /// So we can safely unwrap the Nullable to work with the inner + /// ColumnTuple directly. + IColumn * column_ptr = &info_stack.top().column; + if (info_stack.top().type->isNullable()) + { + auto & nullable_column = assert_cast(*column_ptr); + nullable_column.getNullMapColumn().insertValue(0); + column_ptr = &nullable_column.getNestedColumn(); + } + +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) const auto & tuple_type = assert_cast(*removeNullable(info_stack.top().type)); const auto & nested_types = tuple_type.getElements(); if (size != nested_types.size()) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert MessagePack array with size {} into Tuple column with {} elements", size, nested_types.size()); +<<<<<<< HEAD /// If the type is Nullable, reaching start_array means the value /// is non-null (for nulls, the parser calls visit_nil instead). /// So we can safely unwrap the Nullable to work with the inner @@ -418,6 +434,8 @@ bool MsgPackVisitor::start_array(size_t size) // NOLINT column_ptr = &nullable_column.getNestedColumn(); } +======= +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) ColumnTuple & column_tuple = assert_cast(*column_ptr); /// Push nested columns into stack in reverse order. for (ssize_t i = static_cast(nested_types.size()) - 1; i >= 0; --i) diff --git a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference index 8f1bf22310f9..2c14f9584946 100644 --- a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference +++ b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference @@ -6,25 +6,58 @@ SET engine_file_truncate_on_insert = 1; DROP TABLE IF EXISTS test_nullable_tuple_basic; CREATE TABLE test_nullable_tuple_basic (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_basic VALUES ((1, 'a')), (NULL), ((3, 'c')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; -- Parquet V3 native reader (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; +SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') +\N +(3,'c') +-- Parquet V3 native reader (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_basic; -- Both struct and element nullable: Nullable(Tuple(Nullable(UInt32), String)) DROP TABLE IF EXISTS test_nullable_tuple_both; CREATE TABLE test_nullable_tuple_both (c0 Nullable(Tuple(Nullable(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_both VALUES ((1, 'a')), (NULL), ((NULL, 'c')), ((4, 'd')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; -- Parquet V3 native reader (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader both nullable +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; +SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') +\N +(NULL,'c') +(4,'d') +-- Parquet V3 native reader (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_both; -- Non-nullable struct with nullable elements DROP TABLE IF EXISTS test_nullable_tuple_elem; CREATE TABLE test_nullable_tuple_elem (c0 Tuple(Nullable(UInt32), String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_elem VALUES ((1, 'a')), ((NULL, 'b')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_elem; -- Parquet V3 native reader nullable elements SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)'); +======= +-- Parquet Arrow reader nullable elements +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_elem; +SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') +(NULL,'b') +-- Parquet V3 native reader nullable elements +SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)') SETTINGS input_format_parquet_use_native_reader_v3 = 1; +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) (1,'a') (NULL,'b') DROP TABLE test_nullable_tuple_elem; @@ -32,9 +65,19 @@ DROP TABLE test_nullable_tuple_elem; DROP TABLE IF EXISTS test_nullable_tuple_plain; CREATE TABLE test_nullable_tuple_plain (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_plain VALUES ((1, 'a')), ((2, 'b')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_plain.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_plain; -- Parquet V3 native reader plain SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)'); +======= +-- Parquet Arrow reader plain +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_plain.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_plain; +SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') +(2,'b') +-- Parquet V3 native reader plain +SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 1; +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) (1,'a') (2,'b') DROP TABLE test_nullable_tuple_plain; @@ -42,102 +85,257 @@ DROP TABLE test_nullable_tuple_plain; DROP TABLE IF EXISTS test_nullable_tuple_named; CREATE TABLE test_nullable_tuple_named (c0 Nullable(Tuple(a UInt32, b String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_named VALUES ((1, 'x')), (NULL), ((3, 'z')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; -- Parquet V3 native reader named (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader named +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; +SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'x') +\N +(3,'z') +-- Parquet V3 native reader named (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_named; -- All-NULL column DROP TABLE IF EXISTS test_nullable_tuple_allnull; CREATE TABLE test_nullable_tuple_allnull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_allnull VALUES (NULL), (NULL), (NULL); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; -- Parquet V3 native reader all null (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader all null +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; +SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +\N +\N +\N +-- Parquet V3 native reader all null (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_allnull; -- No-NULL column (nullable type, zero actual NULLs) DROP TABLE IF EXISTS test_nullable_tuple_nonull; CREATE TABLE test_nullable_tuple_nonull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_nonull VALUES ((1, 'a')), ((2, 'b')), ((3, 'c')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; -- Parquet V3 native reader no null (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader no null +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; +SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') +(2,'b') +(3,'c') +-- Parquet V3 native reader no null (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_nonull; -- Single-element tuple DROP TABLE IF EXISTS test_nullable_tuple_single; CREATE TABLE test_nullable_tuple_single (c0 Nullable(Tuple(UInt32))) ENGINE = Memory; INSERT INTO test_nullable_tuple_single VALUES ((1,)), (NULL), ((3,)); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; -- Parquet V3 native reader single (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader single +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; +SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1) +\N +(3) +-- Parquet V3 native reader single (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_single; -- Deeply nested: nullable tuple inside nullable tuple DROP TABLE IF EXISTS test_nullable_tuple_deep; CREATE TABLE test_nullable_tuple_deep (c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_deep VALUES (((1, 'a'), 10)), (NULL), ((NULL, 20)), (((4, 'd'), 40)); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; -- Parquet V3 native reader deep nested (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader deep nested +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; +SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +((1,'a'),10) +\N +(NULL,20) +((4,'d'),40) +-- Parquet V3 native reader deep nested (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_deep; -- Nullable tuple with Array element DROP TABLE IF EXISTS test_nullable_tuple_arr; CREATE TABLE test_nullable_tuple_arr (c0 Nullable(Tuple(Array(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr VALUES (([1, 2], 'a')), (NULL), (([3], 'c')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; -- Parquet V3 native reader array elem (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader array elem +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; +SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +([1,2],'a') +\N +([3],'c') +-- Parquet V3 native reader array elem (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_arr; -- Multiple nullable tuple columns DROP TABLE IF EXISTS test_nullable_tuple_multi; CREATE TABLE test_nullable_tuple_multi (c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_multi VALUES ((1, 'a'), (1.5)), (NULL, (2.5)), ((3, 'c'), NULL); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; -- Parquet V3 native reader multi col (not yet supported) SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_multi; -- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper, not yet supported) +======= +-- Parquet Arrow reader multi col +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; +SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') (1.5) +\N (2.5) +(3,'c') \N +-- Parquet V3 native reader multi col (not yet supported) +SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +DROP TABLE test_nullable_tuple_multi; +-- Schema inference without type hint (works for both readers, but V3 loses struct-level NULL) +DROP TABLE IF EXISTS test_nullable_tuple_infer; +CREATE TABLE test_nullable_tuple_infer (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_infer VALUES ((1, 'a')), (NULL), ((3, 'c')); +-- Parquet Arrow reader infer +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_infer.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_infer; +SELECT c0 FROM file(currentDatabase() || '_04065_infer.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') +\N +(3,'c') +DROP TABLE test_nullable_tuple_infer; +-- Type hint mismatch: file has Nullable(Tuple(...)), read as Tuple(...) (strip nullable, NULLs become defaults) +DROP TABLE IF EXISTS test_nullable_tuple_mismatch1; +CREATE TABLE test_nullable_tuple_mismatch1 (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_mismatch1 VALUES ((1, 'a')), (NULL), ((3, 'c')); +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch1.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch1; +-- Parquet Arrow reader: read nullable file as non-nullable +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch1.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') Tuple(UInt32, String) +(0,'') Tuple(UInt32, String) +(3,'c') Tuple(UInt32, String) +DROP TABLE test_nullable_tuple_mismatch1; +-- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper) +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_mismatch2; CREATE TABLE test_nullable_tuple_mismatch2 (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_mismatch2 VALUES ((1, 'a')), ((2, 'b')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch2; +<<<<<<< HEAD -- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader: read non-nullable file as nullable +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') Nullable(Tuple(UInt32, String)) +(2,'b') Nullable(Tuple(UInt32, String)) +-- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_mismatch2; -- Schema inference: inferred type with toTypeName DROP TABLE IF EXISTS test_nullable_tuple_describe; CREATE TABLE test_nullable_tuple_describe (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_describe VALUES ((1, 'a')), (NULL), ((3, 'c')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_describe; +<<<<<<< HEAD -- Parquet V3 native reader: inferred type (struct-level NULL not supported, becomes (NULL,NULL)) SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet'); +======= +-- Parquet Arrow reader: inferred type +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +(1,'a') Nullable(Tuple(`1` UInt32, `2` String)) +\N Nullable(Tuple(`1` UInt32, `2` String)) +(3,'c') Nullable(Tuple(`1` UInt32, `2` String)) +-- Parquet V3 native reader: inferred type (struct-level NULL not supported, becomes (NULL,NULL)) +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 1; +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) (1,'a') Tuple(\n `1` Nullable(UInt32),\n `2` Nullable(String)) (NULL,NULL) Tuple(\n `1` Nullable(UInt32),\n `2` Nullable(String)) (3,'c') Tuple(\n `1` Nullable(UInt32),\n `2` Nullable(String)) DROP TABLE test_nullable_tuple_describe; +<<<<<<< HEAD -- Array(Nullable(Tuple)) flattened via import_nested +======= +-- Array(Nullable(Tuple)) flattened via import_nested: struct-level NULLs should propagate to elements +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_import_nested; CREATE TABLE test_nullable_tuple_import_nested (c0 Array(Nullable(Tuple(a UInt32, b String)))) ENGINE = Memory; INSERT INTO test_nullable_tuple_import_nested VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_import_nested; +<<<<<<< HEAD -- Parquet V3 native reader import_nested -- This works because V3 reader sees the already-flattened column names (c0.a, c0.b), not the Nullable(Tuple(...)) SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_import_nested = 1; [1,NULL,3] ['a',NULL,'c'] DROP TABLE test_nullable_tuple_import_nested; -- Array(Nullable(Tuple)) without named elements (not yet supported) +======= +-- Parquet Arrow reader import_nested +SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0, input_format_parquet_import_nested = 1; +[1,NULL,3] ['a',NULL,'c'] +-- Parquet V3 native reader import_nested +-- This works because V3 reader sees the already-flattened column names (c0.a, c0.b), not the Nullable(Tuple(...)) +SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_import_nested = 1; +[1,NULL,3] ['a',NULL,'c'] +DROP TABLE test_nullable_tuple_import_nested; +-- Array(Nullable(Tuple)) without named elements: round-trip as a single column, no flattening +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_arr_unnamed; CREATE TABLE test_nullable_tuple_arr_unnamed (c0 Array(Nullable(Tuple(UInt32, String)))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr_unnamed VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_unnamed; +<<<<<<< HEAD -- Parquet V3 native reader unnamed (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_arr_unnamed; -- Array(Nullable(Tuple)) with Array element inside: import_nested flattens +======= +-- Parquet Arrow reader unnamed +SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +[(1,'a'),NULL,(3,'c')] +-- Parquet V3 native reader unnamed (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +DROP TABLE test_nullable_tuple_arr_unnamed; +-- Array(Nullable(Tuple)) with Array element inside: import_nested flattens, Array defaults to [] at null positions +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_arr_nested_elem; CREATE TABLE test_nullable_tuple_arr_nested_elem (c0 Array(Nullable(Tuple(a UInt32, b Array(UInt32))))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr_nested_elem VALUES ([(1, [10, 20]), NULL, (3, [30])]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_nested_elem; +<<<<<<< HEAD -- Parquet V3 native reader import_nested SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_import_nested = 1; +======= +-- Parquet Arrow reader import_nested: scalar becomes Nullable, Array defaults to [] at null struct positions +SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 0, input_format_parquet_import_nested = 1; +[1,NULL,3] [[10,20],[],[30]] +-- Parquet V3 native reader import_nested +SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_import_nested = 1; +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) [1,NULL,3] [[10,20],[],[30]] DROP TABLE test_nullable_tuple_arr_nested_elem; -- LowCardinality(Nullable(String)) hint with no physical nulls in the file: the reader must still wrap the column as nullable @@ -145,8 +343,17 @@ DROP TABLE IF EXISTS test_nullable_tuple_lc_string; CREATE TABLE test_nullable_tuple_lc_string (c0 String) ENGINE = Memory; INSERT INTO test_nullable_tuple_lc_string VALUES ('hello'), ('world'); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_lc_string; +<<<<<<< HEAD -- Parquet V3 native reader: no physical nulls, LowCardinality(Nullable(String)) hint SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))'); +======= +-- Parquet Arrow reader: no physical nulls, LowCardinality(Nullable(String)) hint +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; +hello LowCardinality(Nullable(String)) +world LowCardinality(Nullable(String)) +-- Parquet V3 native reader: no physical nulls, LowCardinality(Nullable(String)) hint +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) hello LowCardinality(Nullable(String)) world LowCardinality(Nullable(String)) DROP TABLE test_nullable_tuple_lc_string; diff --git a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql index 327fb913e874..354080a33b05 100644 --- a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql +++ b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql @@ -11,10 +11,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_basic; CREATE TABLE test_nullable_tuple_basic (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_basic VALUES ((1, 'a')), (NULL), ((3, 'c')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; -- Parquet V3 native reader (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; +SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_basic; @@ -23,10 +32,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_both; CREATE TABLE test_nullable_tuple_both (c0 Nullable(Tuple(Nullable(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_both VALUES ((1, 'a')), (NULL), ((NULL, 'c')), ((4, 'd')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; -- Parquet V3 native reader (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader both nullable +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; +SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_both; @@ -35,10 +53,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_elem; CREATE TABLE test_nullable_tuple_elem (c0 Tuple(Nullable(UInt32), String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_elem VALUES ((1, 'a')), ((NULL, 'b')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_elem; -- Parquet V3 native reader nullable elements SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)'); +======= +-- Parquet Arrow reader nullable elements +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_elem; +SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader nullable elements +SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)') SETTINGS input_format_parquet_use_native_reader_v3 = 1; +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_elem; @@ -47,10 +74,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_plain; CREATE TABLE test_nullable_tuple_plain (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_plain VALUES ((1, 'a')), ((2, 'b')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_plain.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_plain; -- Parquet V3 native reader plain SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)'); +======= +-- Parquet Arrow reader plain +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_plain.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_plain; +SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader plain +SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 1; +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_plain; @@ -59,10 +95,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_named; CREATE TABLE test_nullable_tuple_named (c0 Nullable(Tuple(a UInt32, b String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_named VALUES ((1, 'x')), (NULL), ((3, 'z')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; -- Parquet V3 native reader named (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader named +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; +SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader named (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_named; @@ -71,10 +116,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_allnull; CREATE TABLE test_nullable_tuple_allnull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_allnull VALUES (NULL), (NULL), (NULL); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; -- Parquet V3 native reader all null (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader all null +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; +SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader all null (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_allnull; @@ -83,10 +137,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_nonull; CREATE TABLE test_nullable_tuple_nonull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_nonull VALUES ((1, 'a')), ((2, 'b')), ((3, 'c')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; -- Parquet V3 native reader no null (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader no null +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; +SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader no null (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_nonull; @@ -95,10 +158,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_single; CREATE TABLE test_nullable_tuple_single (c0 Nullable(Tuple(UInt32))) ENGINE = Memory; INSERT INTO test_nullable_tuple_single VALUES ((1,)), (NULL), ((3,)); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; -- Parquet V3 native reader single (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader single +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; +SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader single (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_single; @@ -107,10 +179,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_deep; CREATE TABLE test_nullable_tuple_deep (c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_deep VALUES (((1, 'a'), 10)), (NULL), ((NULL, 20)), (((4, 'd'), 40)); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; -- Parquet V3 native reader deep nested (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader deep nested +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; +SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader deep nested (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_deep; @@ -119,10 +200,19 @@ DROP TABLE IF EXISTS test_nullable_tuple_arr; CREATE TABLE test_nullable_tuple_arr (c0 Nullable(Tuple(Array(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr VALUES (([1, 2], 'a')), (NULL), (([3], 'c')); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; -- Parquet V3 native reader array elem (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader array elem +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; +SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader array elem (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_arr; @@ -131,6 +221,7 @@ DROP TABLE IF EXISTS test_nullable_tuple_multi; CREATE TABLE test_nullable_tuple_multi (c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_multi VALUES ((1, 'a'), (1.5)), (NULL, (2.5)), ((3, 'c'), NULL); +<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; -- Parquet V3 native reader multi col (not yet supported) @@ -139,14 +230,57 @@ SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', DROP TABLE test_nullable_tuple_multi; -- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper, not yet supported) +======= +-- Parquet Arrow reader multi col +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; +SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader multi col (not yet supported) +SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } + +DROP TABLE test_nullable_tuple_multi; + +-- Schema inference without type hint (works for both readers, but V3 loses struct-level NULL) +DROP TABLE IF EXISTS test_nullable_tuple_infer; +CREATE TABLE test_nullable_tuple_infer (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_infer VALUES ((1, 'a')), (NULL), ((3, 'c')); + +-- Parquet Arrow reader infer +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_infer.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_infer; +SELECT c0 FROM file(currentDatabase() || '_04065_infer.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +DROP TABLE test_nullable_tuple_infer; + +-- Type hint mismatch: file has Nullable(Tuple(...)), read as Tuple(...) (strip nullable, NULLs become defaults) +DROP TABLE IF EXISTS test_nullable_tuple_mismatch1; +CREATE TABLE test_nullable_tuple_mismatch1 (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; +INSERT INTO test_nullable_tuple_mismatch1 VALUES ((1, 'a')), (NULL), ((3, 'c')); + +INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch1.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch1; + +-- Parquet Arrow reader: read nullable file as non-nullable +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch1.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +DROP TABLE test_nullable_tuple_mismatch1; + +-- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper) +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_mismatch2; CREATE TABLE test_nullable_tuple_mismatch2 (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_mismatch2 VALUES ((1, 'a')), ((2, 'b')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch2; +<<<<<<< HEAD -- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } +======= +-- Parquet Arrow reader: read non-nullable file as nullable +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_mismatch2; @@ -157,18 +291,31 @@ INSERT INTO test_nullable_tuple_describe VALUES ((1, 'a')), (NULL), ((3, 'c')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_describe; +<<<<<<< HEAD -- Parquet V3 native reader: inferred type (struct-level NULL not supported, becomes (NULL,NULL)) SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet'); DROP TABLE test_nullable_tuple_describe; -- Array(Nullable(Tuple)) flattened via import_nested +======= +-- Parquet Arrow reader: inferred type +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader: inferred type (struct-level NULL not supported, becomes (NULL,NULL)) +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 1; + +DROP TABLE test_nullable_tuple_describe; + +-- Array(Nullable(Tuple)) flattened via import_nested: struct-level NULLs should propagate to elements +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_import_nested; CREATE TABLE test_nullable_tuple_import_nested (c0 Array(Nullable(Tuple(a UInt32, b String)))) ENGINE = Memory; INSERT INTO test_nullable_tuple_import_nested VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_import_nested; +<<<<<<< HEAD -- Parquet V3 native reader import_nested -- This works because V3 reader sees the already-flattened column names (c0.a, c0.b), not the Nullable(Tuple(...)) SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_import_nested = 1; @@ -176,26 +323,58 @@ SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet DROP TABLE test_nullable_tuple_import_nested; -- Array(Nullable(Tuple)) without named elements (not yet supported) +======= +-- Parquet Arrow reader import_nested +SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0, input_format_parquet_import_nested = 1; + +-- Parquet V3 native reader import_nested +-- This works because V3 reader sees the already-flattened column names (c0.a, c0.b), not the Nullable(Tuple(...)) +SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_import_nested = 1; + +DROP TABLE test_nullable_tuple_import_nested; + +-- Array(Nullable(Tuple)) without named elements: round-trip as a single column, no flattening +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_arr_unnamed; CREATE TABLE test_nullable_tuple_arr_unnamed (c0 Array(Nullable(Tuple(UInt32, String)))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr_unnamed VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_unnamed; +<<<<<<< HEAD -- Parquet V3 native reader unnamed (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_arr_unnamed; -- Array(Nullable(Tuple)) with Array element inside: import_nested flattens +======= +-- Parquet Arrow reader unnamed +SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader unnamed (not yet supported) +SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } + +DROP TABLE test_nullable_tuple_arr_unnamed; + +-- Array(Nullable(Tuple)) with Array element inside: import_nested flattens, Array defaults to [] at null positions +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_arr_nested_elem; CREATE TABLE test_nullable_tuple_arr_nested_elem (c0 Array(Nullable(Tuple(a UInt32, b Array(UInt32))))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr_nested_elem VALUES ([(1, [10, 20]), NULL, (3, [30])]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_nested_elem; +<<<<<<< HEAD -- Parquet V3 native reader import_nested SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_import_nested = 1; +======= +-- Parquet Arrow reader import_nested: scalar becomes Nullable, Array defaults to [] at null struct positions +SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 0, input_format_parquet_import_nested = 1; + +-- Parquet V3 native reader import_nested +SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_import_nested = 1; +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_arr_nested_elem; @@ -206,7 +385,15 @@ INSERT INTO test_nullable_tuple_lc_string VALUES ('hello'), ('world'); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_lc_string; +<<<<<<< HEAD -- Parquet V3 native reader: no physical nulls, LowCardinality(Nullable(String)) hint SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))'); +======= +-- Parquet Arrow reader: no physical nulls, LowCardinality(Nullable(String)) hint +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; + +-- Parquet V3 native reader: no physical nulls, LowCardinality(Nullable(String)) hint +SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; +>>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_lc_string; From cb46867686905c4f72dfa66b631883438586311e Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Fri, 26 Jun 2026 10:20:30 +0200 Subject: [PATCH 2/2] Resolve conflicts in cherry-pick of #1802 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CHColumnToArrowColumn.cpp: kept antalya-26.5's architecture (children array + nullBytemapToArrowBitmap + arrow::StructArray::Make) which already incorporates the key changes from the source PR: passing nullptr instead of null_bytemap to nested fillArrowArray calls, and handling struct-level nulls via the null bitmap. The source PR's builder.AppendNull/Append loop is the antalya-26.3 equivalent of antalya-26.5's StructArray::Make with null_bitmap. MsgPackRowInputFormat.cpp: moved the Nullable-unwrapping block to before the tuple_type/size-check lines, matching the source PR's ordering. Test files: kept the antalya-26.5 adaptation (V3-reader-only queries, no input_format_parquet_use_native_reader_v3 variants) because the setting is MAKE_OBSOLETE in antalya-26.5 — V3 reader is always used and the = 0 Arrow reader path cannot be enabled via the setting. --- .../Formats/Impl/CHColumnToArrowColumn.cpp | 13 -- .../Formats/Impl/MsgPackRowInputFormat.cpp | 18 -- ...nside_nullable_parquet_roundtrip.reference | 207 ------------------ ...uple_inside_nullable_parquet_roundtrip.sql | 187 ---------------- 4 files changed, 425 deletions(-) diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 26bb56074795..5a9efde03f5d 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -575,14 +575,9 @@ namespace DB /// Do not propagate the struct-level null_bytemap to child fields. /// In Arrow, struct-level nulls and child-level nulls are independent; /// child values at null struct positions are undefined. -<<<<<<< HEAD auto name = column_name + "." + nested_names[i]; std::shared_ptr nested_arrow_array = fillArrowArray( name, -======= - fillArrowArray( - column_name + "." + nested_names[i], ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) nested_column, nested_types[i], nullptr, builder.field_builder(static_cast(i)), format_name, @@ -593,16 +588,8 @@ namespace DB children.push_back(nested_arrow_array); } -<<<<<<< HEAD auto null_bitmap = nullBytemapToArrowBitmap(null_bytemap, column_name, format_name, start, end); return checkResult(arrow::StructArray::Make(children, builder.type()->fields(), null_bitmap), column_name, format_name); -======= - for (size_t i = start; i != end; ++i) - { - auto status = (null_bytemap && (*null_bytemap)[i]) ? builder.AppendNull() : builder.Append(); - checkStatus(status, column->getName(), format_name); - } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) } template diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index 40803f990a59..8068879f3910 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -401,8 +401,6 @@ bool MsgPackVisitor::start_array(size_t size) // NOLINT } else if (isTuple(removeNullable(info_stack.top().type))) { -<<<<<<< HEAD -======= /// If the type is Nullable, reaching start_array means the value /// is non-null (for nulls, the parser calls visit_nil instead). /// So we can safely unwrap the Nullable to work with the inner @@ -415,27 +413,11 @@ bool MsgPackVisitor::start_array(size_t size) // NOLINT column_ptr = &nullable_column.getNestedColumn(); } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) const auto & tuple_type = assert_cast(*removeNullable(info_stack.top().type)); const auto & nested_types = tuple_type.getElements(); if (size != nested_types.size()) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert MessagePack array with size {} into Tuple column with {} elements", size, nested_types.size()); -<<<<<<< HEAD - /// If the type is Nullable, reaching start_array means the value - /// is non-null (for nulls, the parser calls visit_nil instead). - /// So we can safely unwrap the Nullable to work with the inner - /// ColumnTuple directly. - IColumn * column_ptr = &info_stack.top().column; - if (info_stack.top().type->isNullable()) - { - auto & nullable_column = assert_cast(*column_ptr); - nullable_column.getNullMapColumn().insertValue(0); - column_ptr = &nullable_column.getNestedColumn(); - } - -======= ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) ColumnTuple & column_tuple = assert_cast(*column_ptr); /// Push nested columns into stack in reverse order. for (ssize_t i = static_cast(nested_types.size()) - 1; i >= 0; --i) diff --git a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference index 2c14f9584946..8f1bf22310f9 100644 --- a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference +++ b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.reference @@ -6,58 +6,25 @@ SET engine_file_truncate_on_insert = 1; DROP TABLE IF EXISTS test_nullable_tuple_basic; CREATE TABLE test_nullable_tuple_basic (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_basic VALUES ((1, 'a')), (NULL), ((3, 'c')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; -- Parquet V3 native reader (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; -SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') -\N -(3,'c') --- Parquet V3 native reader (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_basic; -- Both struct and element nullable: Nullable(Tuple(Nullable(UInt32), String)) DROP TABLE IF EXISTS test_nullable_tuple_both; CREATE TABLE test_nullable_tuple_both (c0 Nullable(Tuple(Nullable(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_both VALUES ((1, 'a')), (NULL), ((NULL, 'c')), ((4, 'd')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; -- Parquet V3 native reader (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader both nullable -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; -SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') -\N -(NULL,'c') -(4,'d') --- Parquet V3 native reader (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_both; -- Non-nullable struct with nullable elements DROP TABLE IF EXISTS test_nullable_tuple_elem; CREATE TABLE test_nullable_tuple_elem (c0 Tuple(Nullable(UInt32), String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_elem VALUES ((1, 'a')), ((NULL, 'b')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_elem; -- Parquet V3 native reader nullable elements SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)'); -======= --- Parquet Arrow reader nullable elements -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_elem; -SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') -(NULL,'b') --- Parquet V3 native reader nullable elements -SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)') SETTINGS input_format_parquet_use_native_reader_v3 = 1; ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) (1,'a') (NULL,'b') DROP TABLE test_nullable_tuple_elem; @@ -65,19 +32,9 @@ DROP TABLE test_nullable_tuple_elem; DROP TABLE IF EXISTS test_nullable_tuple_plain; CREATE TABLE test_nullable_tuple_plain (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_plain VALUES ((1, 'a')), ((2, 'b')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_plain.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_plain; -- Parquet V3 native reader plain SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)'); -======= --- Parquet Arrow reader plain -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_plain.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_plain; -SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') -(2,'b') --- Parquet V3 native reader plain -SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 1; ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) (1,'a') (2,'b') DROP TABLE test_nullable_tuple_plain; @@ -85,257 +42,102 @@ DROP TABLE test_nullable_tuple_plain; DROP TABLE IF EXISTS test_nullable_tuple_named; CREATE TABLE test_nullable_tuple_named (c0 Nullable(Tuple(a UInt32, b String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_named VALUES ((1, 'x')), (NULL), ((3, 'z')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; -- Parquet V3 native reader named (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader named -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; -SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'x') -\N -(3,'z') --- Parquet V3 native reader named (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_named; -- All-NULL column DROP TABLE IF EXISTS test_nullable_tuple_allnull; CREATE TABLE test_nullable_tuple_allnull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_allnull VALUES (NULL), (NULL), (NULL); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; -- Parquet V3 native reader all null (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader all null -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; -SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -\N -\N -\N --- Parquet V3 native reader all null (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_allnull; -- No-NULL column (nullable type, zero actual NULLs) DROP TABLE IF EXISTS test_nullable_tuple_nonull; CREATE TABLE test_nullable_tuple_nonull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_nonull VALUES ((1, 'a')), ((2, 'b')), ((3, 'c')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; -- Parquet V3 native reader no null (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader no null -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; -SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') -(2,'b') -(3,'c') --- Parquet V3 native reader no null (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_nonull; -- Single-element tuple DROP TABLE IF EXISTS test_nullable_tuple_single; CREATE TABLE test_nullable_tuple_single (c0 Nullable(Tuple(UInt32))) ENGINE = Memory; INSERT INTO test_nullable_tuple_single VALUES ((1,)), (NULL), ((3,)); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; -- Parquet V3 native reader single (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader single -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; -SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1) -\N -(3) --- Parquet V3 native reader single (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_single; -- Deeply nested: nullable tuple inside nullable tuple DROP TABLE IF EXISTS test_nullable_tuple_deep; CREATE TABLE test_nullable_tuple_deep (c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_deep VALUES (((1, 'a'), 10)), (NULL), ((NULL, 20)), (((4, 'd'), 40)); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; -- Parquet V3 native reader deep nested (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader deep nested -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; -SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -((1,'a'),10) -\N -(NULL,20) -((4,'d'),40) --- Parquet V3 native reader deep nested (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_deep; -- Nullable tuple with Array element DROP TABLE IF EXISTS test_nullable_tuple_arr; CREATE TABLE test_nullable_tuple_arr (c0 Nullable(Tuple(Array(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr VALUES (([1, 2], 'a')), (NULL), (([3], 'c')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; -- Parquet V3 native reader array elem (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader array elem -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; -SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -([1,2],'a') -\N -([3],'c') --- Parquet V3 native reader array elem (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_arr; -- Multiple nullable tuple columns DROP TABLE IF EXISTS test_nullable_tuple_multi; CREATE TABLE test_nullable_tuple_multi (c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_multi VALUES ((1, 'a'), (1.5)), (NULL, (2.5)), ((3, 'c'), NULL); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; -- Parquet V3 native reader multi col (not yet supported) SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_multi; -- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper, not yet supported) -======= --- Parquet Arrow reader multi col -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; -SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') (1.5) -\N (2.5) -(3,'c') \N --- Parquet V3 native reader multi col (not yet supported) -SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } -DROP TABLE test_nullable_tuple_multi; --- Schema inference without type hint (works for both readers, but V3 loses struct-level NULL) -DROP TABLE IF EXISTS test_nullable_tuple_infer; -CREATE TABLE test_nullable_tuple_infer (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; -INSERT INTO test_nullable_tuple_infer VALUES ((1, 'a')), (NULL), ((3, 'c')); --- Parquet Arrow reader infer -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_infer.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_infer; -SELECT c0 FROM file(currentDatabase() || '_04065_infer.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') -\N -(3,'c') -DROP TABLE test_nullable_tuple_infer; --- Type hint mismatch: file has Nullable(Tuple(...)), read as Tuple(...) (strip nullable, NULLs become defaults) -DROP TABLE IF EXISTS test_nullable_tuple_mismatch1; -CREATE TABLE test_nullable_tuple_mismatch1 (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; -INSERT INTO test_nullable_tuple_mismatch1 VALUES ((1, 'a')), (NULL), ((3, 'c')); -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch1.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch1; --- Parquet Arrow reader: read nullable file as non-nullable -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch1.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') Tuple(UInt32, String) -(0,'') Tuple(UInt32, String) -(3,'c') Tuple(UInt32, String) -DROP TABLE test_nullable_tuple_mismatch1; --- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper) ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_mismatch2; CREATE TABLE test_nullable_tuple_mismatch2 (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_mismatch2 VALUES ((1, 'a')), ((2, 'b')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch2; -<<<<<<< HEAD -- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader: read non-nullable file as nullable -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') Nullable(Tuple(UInt32, String)) -(2,'b') Nullable(Tuple(UInt32, String)) --- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_mismatch2; -- Schema inference: inferred type with toTypeName DROP TABLE IF EXISTS test_nullable_tuple_describe; CREATE TABLE test_nullable_tuple_describe (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_describe VALUES ((1, 'a')), (NULL), ((3, 'c')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_describe; -<<<<<<< HEAD -- Parquet V3 native reader: inferred type (struct-level NULL not supported, becomes (NULL,NULL)) SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet'); -======= --- Parquet Arrow reader: inferred type -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -(1,'a') Nullable(Tuple(`1` UInt32, `2` String)) -\N Nullable(Tuple(`1` UInt32, `2` String)) -(3,'c') Nullable(Tuple(`1` UInt32, `2` String)) --- Parquet V3 native reader: inferred type (struct-level NULL not supported, becomes (NULL,NULL)) -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 1; ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) (1,'a') Tuple(\n `1` Nullable(UInt32),\n `2` Nullable(String)) (NULL,NULL) Tuple(\n `1` Nullable(UInt32),\n `2` Nullable(String)) (3,'c') Tuple(\n `1` Nullable(UInt32),\n `2` Nullable(String)) DROP TABLE test_nullable_tuple_describe; -<<<<<<< HEAD -- Array(Nullable(Tuple)) flattened via import_nested -======= --- Array(Nullable(Tuple)) flattened via import_nested: struct-level NULLs should propagate to elements ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_import_nested; CREATE TABLE test_nullable_tuple_import_nested (c0 Array(Nullable(Tuple(a UInt32, b String)))) ENGINE = Memory; INSERT INTO test_nullable_tuple_import_nested VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_import_nested; -<<<<<<< HEAD -- Parquet V3 native reader import_nested -- This works because V3 reader sees the already-flattened column names (c0.a, c0.b), not the Nullable(Tuple(...)) SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_import_nested = 1; [1,NULL,3] ['a',NULL,'c'] DROP TABLE test_nullable_tuple_import_nested; -- Array(Nullable(Tuple)) without named elements (not yet supported) -======= --- Parquet Arrow reader import_nested -SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0, input_format_parquet_import_nested = 1; -[1,NULL,3] ['a',NULL,'c'] --- Parquet V3 native reader import_nested --- This works because V3 reader sees the already-flattened column names (c0.a, c0.b), not the Nullable(Tuple(...)) -SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_import_nested = 1; -[1,NULL,3] ['a',NULL,'c'] -DROP TABLE test_nullable_tuple_import_nested; --- Array(Nullable(Tuple)) without named elements: round-trip as a single column, no flattening ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_arr_unnamed; CREATE TABLE test_nullable_tuple_arr_unnamed (c0 Array(Nullable(Tuple(UInt32, String)))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr_unnamed VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_unnamed; -<<<<<<< HEAD -- Parquet V3 native reader unnamed (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_arr_unnamed; -- Array(Nullable(Tuple)) with Array element inside: import_nested flattens -======= --- Parquet Arrow reader unnamed -SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -[(1,'a'),NULL,(3,'c')] --- Parquet V3 native reader unnamed (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } -DROP TABLE test_nullable_tuple_arr_unnamed; --- Array(Nullable(Tuple)) with Array element inside: import_nested flattens, Array defaults to [] at null positions ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_arr_nested_elem; CREATE TABLE test_nullable_tuple_arr_nested_elem (c0 Array(Nullable(Tuple(a UInt32, b Array(UInt32))))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr_nested_elem VALUES ([(1, [10, 20]), NULL, (3, [30])]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_nested_elem; -<<<<<<< HEAD -- Parquet V3 native reader import_nested SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_import_nested = 1; -======= --- Parquet Arrow reader import_nested: scalar becomes Nullable, Array defaults to [] at null struct positions -SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 0, input_format_parquet_import_nested = 1; -[1,NULL,3] [[10,20],[],[30]] --- Parquet V3 native reader import_nested -SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_import_nested = 1; ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) [1,NULL,3] [[10,20],[],[30]] DROP TABLE test_nullable_tuple_arr_nested_elem; -- LowCardinality(Nullable(String)) hint with no physical nulls in the file: the reader must still wrap the column as nullable @@ -343,17 +145,8 @@ DROP TABLE IF EXISTS test_nullable_tuple_lc_string; CREATE TABLE test_nullable_tuple_lc_string (c0 String) ENGINE = Memory; INSERT INTO test_nullable_tuple_lc_string VALUES ('hello'), ('world'); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_lc_string; -<<<<<<< HEAD -- Parquet V3 native reader: no physical nulls, LowCardinality(Nullable(String)) hint SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))'); -======= --- Parquet Arrow reader: no physical nulls, LowCardinality(Nullable(String)) hint -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; -hello LowCardinality(Nullable(String)) -world LowCardinality(Nullable(String)) --- Parquet V3 native reader: no physical nulls, LowCardinality(Nullable(String)) hint -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) hello LowCardinality(Nullable(String)) world LowCardinality(Nullable(String)) DROP TABLE test_nullable_tuple_lc_string; diff --git a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql index 354080a33b05..327fb913e874 100644 --- a/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql +++ b/tests/queries/0_stateless/04065_tuple_inside_nullable_parquet_roundtrip.sql @@ -11,19 +11,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_basic; CREATE TABLE test_nullable_tuple_basic (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_basic VALUES ((1, 'a')), (NULL), ((3, 'c')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; -- Parquet V3 native reader (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_basic; -SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_basic; @@ -32,19 +23,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_both; CREATE TABLE test_nullable_tuple_both (c0 Nullable(Tuple(Nullable(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_both VALUES ((1, 'a')), (NULL), ((NULL, 'c')), ((4, 'd')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; -- Parquet V3 native reader (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader both nullable -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_both.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_both; -SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_both.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_both; @@ -53,19 +35,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_elem; CREATE TABLE test_nullable_tuple_elem (c0 Tuple(Nullable(UInt32), String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_elem VALUES ((1, 'a')), ((NULL, 'b')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_elem; -- Parquet V3 native reader nullable elements SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)'); -======= --- Parquet Arrow reader nullable elements -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_elem; -SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader nullable elements -SELECT c0 FROM file(currentDatabase() || '_04065_elem.parquet', 'Parquet', 'c0 Tuple(Nullable(UInt32), String)') SETTINGS input_format_parquet_use_native_reader_v3 = 1; ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_elem; @@ -74,19 +47,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_plain; CREATE TABLE test_nullable_tuple_plain (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_plain VALUES ((1, 'a')), ((2, 'b')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_plain.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_plain; -- Parquet V3 native reader plain SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)'); -======= --- Parquet Arrow reader plain -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_plain.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_plain; -SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader plain -SELECT c0 FROM file(currentDatabase() || '_04065_plain.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 1; ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_plain; @@ -95,19 +59,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_named; CREATE TABLE test_nullable_tuple_named (c0 Nullable(Tuple(a UInt32, b String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_named VALUES ((1, 'x')), (NULL), ((3, 'z')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; -- Parquet V3 native reader named (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader named -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SELECT c0 FROM test_nullable_tuple_named; -SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader named (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_named.parquet', 'Parquet', 'c0 Nullable(Tuple(a UInt32, b String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_named; @@ -116,19 +71,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_allnull; CREATE TABLE test_nullable_tuple_allnull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_allnull VALUES (NULL), (NULL), (NULL); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; -- Parquet V3 native reader all null (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader all null -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_allnull; -SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader all null (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_allnull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_allnull; @@ -137,19 +83,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_nonull; CREATE TABLE test_nullable_tuple_nonull (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_nonull VALUES ((1, 'a')), ((2, 'b')), ((3, 'c')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; -- Parquet V3 native reader no null (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader no null -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SELECT c0 FROM test_nullable_tuple_nonull; -SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader no null (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_nonull.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_nonull; @@ -158,19 +95,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_single; CREATE TABLE test_nullable_tuple_single (c0 Nullable(Tuple(UInt32))) ENGINE = Memory; INSERT INTO test_nullable_tuple_single VALUES ((1,)), (NULL), ((3,)); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; -- Parquet V3 native reader single (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader single -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SELECT c0 FROM test_nullable_tuple_single; -SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader single (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_single.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_single; @@ -179,19 +107,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_deep; CREATE TABLE test_nullable_tuple_deep (c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_deep VALUES (((1, 'a'), 10)), (NULL), ((NULL, 20)), (((4, 'd'), 40)); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; -- Parquet V3 native reader deep nested (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader deep nested -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_deep.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_deep; -SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader deep nested (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_deep.parquet', 'Parquet', 'c0 Nullable(Tuple(Nullable(Tuple(UInt32, String)), UInt64))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_deep; @@ -200,19 +119,10 @@ DROP TABLE IF EXISTS test_nullable_tuple_arr; CREATE TABLE test_nullable_tuple_arr (c0 Nullable(Tuple(Array(UInt32), String))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr VALUES (([1, 2], 'a')), (NULL), (([3], 'c')); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; -- Parquet V3 native reader array elem (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader array elem -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr; -SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader array elem (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_arr.parquet', 'Parquet', 'c0 Nullable(Tuple(Array(UInt32), String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_arr; @@ -221,7 +131,6 @@ DROP TABLE IF EXISTS test_nullable_tuple_multi; CREATE TABLE test_nullable_tuple_multi (c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))) ENGINE = Memory; INSERT INTO test_nullable_tuple_multi VALUES ((1, 'a'), (1.5)), (NULL, (2.5)), ((3, 'c'), NULL); -<<<<<<< HEAD INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; -- Parquet V3 native reader multi col (not yet supported) @@ -230,57 +139,14 @@ SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', DROP TABLE test_nullable_tuple_multi; -- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper, not yet supported) -======= --- Parquet Arrow reader multi col -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_multi.parquet', 'Parquet') SELECT c0, c1 FROM test_nullable_tuple_multi; -SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader multi col (not yet supported) -SELECT c0, c1 FROM file(currentDatabase() || '_04065_multi.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String)), c1 Nullable(Tuple(Float64))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } - -DROP TABLE test_nullable_tuple_multi; - --- Schema inference without type hint (works for both readers, but V3 loses struct-level NULL) -DROP TABLE IF EXISTS test_nullable_tuple_infer; -CREATE TABLE test_nullable_tuple_infer (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; -INSERT INTO test_nullable_tuple_infer VALUES ((1, 'a')), (NULL), ((3, 'c')); - --- Parquet Arrow reader infer -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_infer.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_infer; -SELECT c0 FROM file(currentDatabase() || '_04065_infer.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - -DROP TABLE test_nullable_tuple_infer; - --- Type hint mismatch: file has Nullable(Tuple(...)), read as Tuple(...) (strip nullable, NULLs become defaults) -DROP TABLE IF EXISTS test_nullable_tuple_mismatch1; -CREATE TABLE test_nullable_tuple_mismatch1 (c0 Nullable(Tuple(UInt32, String))) ENGINE = Memory; -INSERT INTO test_nullable_tuple_mismatch1 VALUES ((1, 'a')), (NULL), ((3, 'c')); - -INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch1.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch1; - --- Parquet Arrow reader: read nullable file as non-nullable -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch1.parquet', 'Parquet', 'c0 Tuple(UInt32, String)') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - -DROP TABLE test_nullable_tuple_mismatch1; - --- Type hint mismatch: file has Tuple(...), read as Nullable(Tuple(...)) (add nullable wrapper) ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_mismatch2; CREATE TABLE test_nullable_tuple_mismatch2 (c0 Tuple(UInt32, String)) ENGINE = Memory; INSERT INTO test_nullable_tuple_mismatch2 VALUES ((1, 'a')), ((2, 'b')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_mismatch2; -<<<<<<< HEAD -- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))'); -- { serverError TYPE_MISMATCH } -======= --- Parquet Arrow reader: read non-nullable file as nullable -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader: read non-nullable file as nullable (not yet supported) -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_mismatch2.parquet', 'Parquet', 'c0 Nullable(Tuple(UInt32, String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_mismatch2; @@ -291,31 +157,18 @@ INSERT INTO test_nullable_tuple_describe VALUES ((1, 'a')), (NULL), ((3, 'c')); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_describe; -<<<<<<< HEAD -- Parquet V3 native reader: inferred type (struct-level NULL not supported, becomes (NULL,NULL)) SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet'); DROP TABLE test_nullable_tuple_describe; -- Array(Nullable(Tuple)) flattened via import_nested -======= --- Parquet Arrow reader: inferred type -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader: inferred type (struct-level NULL not supported, becomes (NULL,NULL)) -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_describe.parquet', 'Parquet') SETTINGS input_format_parquet_use_native_reader_v3 = 1; - -DROP TABLE test_nullable_tuple_describe; - --- Array(Nullable(Tuple)) flattened via import_nested: struct-level NULLs should propagate to elements ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_import_nested; CREATE TABLE test_nullable_tuple_import_nested (c0 Array(Nullable(Tuple(a UInt32, b String)))) ENGINE = Memory; INSERT INTO test_nullable_tuple_import_nested VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_import_nested; -<<<<<<< HEAD -- Parquet V3 native reader import_nested -- This works because V3 reader sees the already-flattened column names (c0.a, c0.b), not the Nullable(Tuple(...)) SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_import_nested = 1; @@ -323,58 +176,26 @@ SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet DROP TABLE test_nullable_tuple_import_nested; -- Array(Nullable(Tuple)) without named elements (not yet supported) -======= --- Parquet Arrow reader import_nested -SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0, input_format_parquet_import_nested = 1; - --- Parquet V3 native reader import_nested --- This works because V3 reader sees the already-flattened column names (c0.a, c0.b), not the Nullable(Tuple(...)) -SELECT * FROM file(currentDatabase() || '_04065_import_nested.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_import_nested = 1; - -DROP TABLE test_nullable_tuple_import_nested; - --- Array(Nullable(Tuple)) without named elements: round-trip as a single column, no flattening ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_arr_unnamed; CREATE TABLE test_nullable_tuple_arr_unnamed (c0 Array(Nullable(Tuple(UInt32, String)))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr_unnamed VALUES ([(1, 'a'), NULL, (3, 'c')]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_unnamed; -<<<<<<< HEAD -- Parquet V3 native reader unnamed (not yet supported) SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))'); -- { serverError TYPE_MISMATCH } DROP TABLE test_nullable_tuple_arr_unnamed; -- Array(Nullable(Tuple)) with Array element inside: import_nested flattens -======= --- Parquet Arrow reader unnamed -SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader unnamed (not yet supported) -SELECT c0 FROM file(currentDatabase() || '_04065_arr_unnamed.parquet', 'Parquet', 'c0 Array(Nullable(Tuple(UInt32, String)))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; -- { serverError TYPE_MISMATCH } - -DROP TABLE test_nullable_tuple_arr_unnamed; - --- Array(Nullable(Tuple)) with Array element inside: import_nested flattens, Array defaults to [] at null positions ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE IF EXISTS test_nullable_tuple_arr_nested_elem; CREATE TABLE test_nullable_tuple_arr_nested_elem (c0 Array(Nullable(Tuple(a UInt32, b Array(UInt32))))) ENGINE = Memory; INSERT INTO test_nullable_tuple_arr_nested_elem VALUES ([(1, [10, 20]), NULL, (3, [30])]); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_arr_nested_elem; -<<<<<<< HEAD -- Parquet V3 native reader import_nested SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_import_nested = 1; -======= --- Parquet Arrow reader import_nested: scalar becomes Nullable, Array defaults to [] at null struct positions -SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 0, input_format_parquet_import_nested = 1; - --- Parquet V3 native reader import_nested -SELECT * FROM file(currentDatabase() || '_04065_arr_nested_elem.parquet', 'Parquet', '`c0.a` Array(Nullable(UInt32)), `c0.b` Array(Array(UInt32))') SETTINGS input_format_parquet_use_native_reader_v3 = 1, input_format_parquet_import_nested = 1; ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_arr_nested_elem; @@ -385,15 +206,7 @@ INSERT INTO test_nullable_tuple_lc_string VALUES ('hello'), ('world'); INSERT INTO TABLE FUNCTION file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet') SELECT c0 FROM test_nullable_tuple_lc_string; -<<<<<<< HEAD -- Parquet V3 native reader: no physical nulls, LowCardinality(Nullable(String)) hint SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))'); -======= --- Parquet Arrow reader: no physical nulls, LowCardinality(Nullable(String)) hint -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 0; - --- Parquet V3 native reader: no physical nulls, LowCardinality(Nullable(String)) hint -SELECT c0, toTypeName(c0) FROM file(currentDatabase() || '_04065_lc_str.parquet', 'Parquet', 'c0 LowCardinality(Nullable(String))') SETTINGS input_format_parquet_use_native_reader_v3 = 1; ->>>>>>> 57b8b335336 (Merge pull request #1802 from Altinity/feature/antalya-26.3/ClickHouse-ClickHouse-pr-101272) DROP TABLE test_nullable_tuple_lc_string;