From 26004b5dbc0bac54d5f038a8ed609b229154412e Mon Sep 17 00:00:00 2001 From: ragupta Date: Thu, 8 Jan 2026 18:00:43 +0530 Subject: [PATCH 01/19] HIVE-29376:Using partition spec in DESC FORMATTED sql is unsupported for Iceberg table --- .../TestHiveIcebergStorageHandlerNoScan.java | 27 +----- .../negative/desc_ice_tbl_partial_part_spec.q | 4 + .../negative/desc_ice_tbl_wrong_part_spec.q | 4 + .../queries/positive/desc_ice_tbl_part_spec.q | 9 ++ .../desc_ice_tbl_partial_part_spec.q.out | 23 +++++ .../desc_ice_tbl_wrong_part_spec.q.out | 23 +++++ .../alter_multi_part_table_to_iceberg.q.out | 33 +++++++ .../alter_part_table_to_iceberg.q.out | 20 ++++ .../ctas_iceberg_partitioned_orc.q.out | 5 + .../positive/desc_ice_tbl_part_spec.q.out | 94 +++++++++++++++++++ .../positive/describe_iceberg_table.q.out | 24 +++++ .../positive/iceberg_drop_column.q.out | 5 + .../iceberg_insert_into_partition.q.out | 26 +++++ ...erg_insert_into_partition_transforms.q.out | 24 +++++ ...insert_into_partition_with_evolution.q.out | 4 + .../iceberg_insert_overwrite_partition.q.out | 25 +++++ ...nsert_overwrite_partition_transforms.q.out | 16 ++++ .../llap/hadoop_catalog_create_table.q.out | 25 +++++ ...major_compaction_partition_evolution.q.out | 10 ++ ...ajor_compaction_partition_evolution2.q.out | 8 ++ ...mpaction_partition_evolution_ordered.q.out | 8 ++ ...tition_evolution_w_dyn_spec_w_filter.q.out | 10 ++ ...rtition_evolution_w_id_spec_w_filter.q.out | 10 ++ ...iceberg_major_compaction_partitioned.q.out | 16 ++++ ...rg_major_compaction_schema_evolution.q.out | 8 ++ ...rg_major_compaction_single_partition.q.out | 18 ++++ ...tion_single_partition_with_evolution.q.out | 30 ++++++ ...ion_single_partition_with_evolution2.q.out | 16 ++++ .../iceberg_minor_compaction_bucket.q.out | 8 ++ ...minor_compaction_partition_evolution.q.out | 12 +++ .../llap/vectorized_iceberg_read_mixed.q.out | 5 + .../llap/vectorized_iceberg_read_orc.q.out | 5 + .../vectorized_iceberg_read_parquet.q.out | 5 + .../positive/mv_iceberg_partitioned_orc.q.out | 8 ++ .../mv_iceberg_partitioned_orc2.q.out | 10 ++ .../src/test/results/positive/row_count.q.out | 10 ++ .../positive/show_partitions_test.q.out | 5 + .../truncate_partitioned_iceberg_table.q.out | 8 ++ .../vectorized_iceberg_read_mixed.q.out | 5 + .../vectorized_iceberg_read_orc.q.out | 5 + .../vectorized_iceberg_read_parquet.q.out | 5 + .../table/info/desc/DescTableAnalyzer.java | 16 +++- .../table/info/desc/DescTableOperation.java | 10 +- .../formatter/TextDescTableFormatter.java | 6 +- .../hive/ql/metadata/DummyPartition.java | 4 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 3 +- 46 files changed, 627 insertions(+), 28 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q create mode 100644 iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q create mode 100644 iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out create mode 100644 iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out create mode 100644 iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java index 3d12cfb1179a..343fa4429669 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java @@ -959,8 +959,8 @@ public void testCreatePartitionedTableWithColumnComments() { List rows = shell.executeStatement("DESCRIBE default.partitioned_with_comment_table"); List columns = icebergTable.schema().columns(); - // The partition transform information is 3 extra lines, and 2 more line for the columns - Assert.assertEquals(columns.size() + 5, rows.size()); + // The partition transform information and partition information is 6 extra lines, and 4 more line for the columns + Assert.assertEquals(columns.size() + 10, rows.size()); for (int i = 0; i < columns.size(); i++) { Types.NestedField field = columns.get(i); Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(), @@ -1323,8 +1323,8 @@ public void testAlterTableRenamePartitionColumn() throws Exception { shell.executeStatement("ALTER TABLE default.customers SET PARTITION SPEC (region, city)"); List result = shell.executeStatement("DESCRIBE default.customers"); - Assert.assertArrayEquals(new String[] {"region", "IDENTITY", null}, result.get(8)); - Assert.assertArrayEquals(new String[] {"city", "IDENTITY", null}, result.get(9)); + Assert.assertArrayEquals(new String[] {"region", "IDENTITY", null}, result.get(14)); + Assert.assertArrayEquals(new String[] {"city", "IDENTITY", null}, result.get(15)); } @Test @@ -1490,25 +1490,6 @@ public void testMetaHookWithUndefinedAlterOperationType() throws Exception { metaHook.commitAlterTable(hmsTable, environmentContext); } - @Test - public void testCommandsWithPartitionClauseThrow() { - TableIdentifier target = TableIdentifier.of("default", "target"); - PartitionSpec spec = PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA) - .identity("last_name").build(); - testTables.createTable(shell, target.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, - spec, FileFormat.PARQUET, ImmutableList.of()); - - String[] commands = { - "DESCRIBE target PARTITION (last_name='Johnson')" - }; - - for (String command : commands) { - Assertions.assertThatThrownBy(() -> shell.executeStatement(command)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Using partition spec in query is unsupported"); - } - } - @Test public void testAuthzURIMasked() throws TException, URISyntaxException, InterruptedException { testAuthzURI(true); diff --git a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q new file mode 100644 index 000000000000..577785fd6956 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q @@ -0,0 +1,4 @@ +drop table if exists ice_t; +create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; +insert into table ice_t values( 5, "hello5" ,6, "hello6"); +desc formatted ice_t PARTITION(c=6); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q new file mode 100644 index 000000000000..974e61c14c1b --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q @@ -0,0 +1,4 @@ +drop table if exists ice_t; +create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; +insert into table ice_t values( 5, "hello5" ,6, "hello6"); +desc formatted ice_t PARTITION(c=6, d="hello"); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q new file mode 100644 index 000000000000..c020c08997a1 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q @@ -0,0 +1,9 @@ +drop table if exists ice_t; +create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; + +insert into table ice_t values( 1, "hello1" ,2, "hello2"); +insert into table ice_t values( 3, "hello3" ,4, "hello4"); +insert into table ice_t values( 5, "hello5" ,6, "hello6"); + +desc extended ice_t PARTITION(c=6,d="hello6"); +desc formatted ice_t PARTITION(c=6,d="hello6"); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out new file mode 100644 index 000000000000..0165c68d6245 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out @@ -0,0 +1,23 @@ +PREHOOK: query: drop table if exists ice_t +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists ice_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +FAILED: SemanticException [Error 10006]: Partition not found {c=6} diff --git a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out new file mode 100644 index 000000000000..fb6e7f7acf45 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out @@ -0,0 +1,23 @@ +PREHOOK: query: drop table if exists ice_t +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists ice_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +FAILED: SemanticException [Error 10006]: Partition not found {c=6, d=hello} diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out index 118552ac4d5d..910e48e4214e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out @@ -180,6 +180,11 @@ a int b string c string +# Partition Information +# col_name data_type comment +b string Transform: identity +c string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -451,6 +456,11 @@ a int b string c string +# Partition Information +# col_name data_type comment +b string Transform: identity +c string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -722,6 +732,11 @@ a int b string c string +# Partition Information +# col_name data_type comment +b string Transform: identity +c string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1055,6 +1070,12 @@ b double c int d string +# Partition Information +# col_name data_type comment +b double Transform: identity +c int Transform: identity +d string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1496,6 +1517,12 @@ b double c int d string +# Partition Information +# col_name data_type comment +b double Transform: identity +c int Transform: identity +d string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1937,6 +1964,12 @@ b double c int d string +# Partition Information +# col_name data_type comment +b double Transform: identity +c int Transform: identity +d string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out index 2e55bbd42b7c..55bfee6eb031 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out @@ -139,6 +139,10 @@ POSTHOOK: Input: default@tbl_orc a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -413,6 +417,10 @@ POSTHOOK: Input: default@tbl_parquet a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -764,6 +772,10 @@ POSTHOOK: Input: default@tbl_parquet_int a int b int +# Partition Information +# col_name data_type comment +b int Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1115,6 +1127,10 @@ POSTHOOK: Input: default@tbl_parquet_double a int b double +# Partition Information +# col_name data_type comment +b double Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1412,6 +1428,10 @@ POSTHOOK: Input: default@tbl_avro a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out index c0a81219df6d..0d1700ff07a9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out @@ -277,6 +277,11 @@ a int b string c int +# Partition Information +# col_name data_type comment +a int Transform: bucket[16] +b string Transform: truncate[3] + # Partition Transform Information # col_name transform_type a BUCKET[16] diff --git a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out new file mode 100644 index 000000000000..f62c7309ecf2 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out @@ -0,0 +1,94 @@ +PREHOOK: query: drop table if exists ice_t +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists ice_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: desc extended ice_t PARTITION(c=6,d="hello6") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: desc extended ice_t PARTITION(c=6,d="hello6") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY + +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_t PARTITION(c=6,d="hello6") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: desc formatted ice_t PARTITION(c=6,d="hello6") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY + +# Detailed Partition Information +Partition Value: [6, hello6] +Database: default +Table: ice_t +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out index 09ef4d1ac96c..fb1cdbcaf12a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out @@ -117,6 +117,16 @@ truncate_field string bucket_field int identity_field int +# Partition Information +# col_name data_type comment +year_field date Transform: year +month_field date Transform: month +day_field date Transform: day +hour_field timestamp Transform: hour +truncate_field string Transform: truncate[2] +bucket_field int Transform: bucket[2] +identity_field int Transform: identity + # Partition Transform Information # col_name transform_type year_field YEAR @@ -180,6 +190,16 @@ truncate_field string bucket_field int identity_field int +# Partition Information +# col_name data_type comment +year_field date Transform: year +month_field date Transform: month +day_field date Transform: day +hour_field timestamp Transform: hour +truncate_field string Transform: truncate[2] +bucket_field int Transform: bucket[2] +identity_field int Transform: identity + # Partition Transform Information # col_name transform_type year_field YEAR @@ -237,6 +257,10 @@ POSTHOOK: Input: default@ice_t_identity_part a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out index c7e7c4e4acf7..74d891eefc1e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out @@ -30,6 +30,11 @@ intcol int pcol string datecol date +# Partition Information +# col_name data_type comment +pcol string Transform: identity +datecol date Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index 000b8b0ba819..c68cb256cd60 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -481,6 +481,10 @@ strcol string intcol int pcol int +# Partition Information +# col_name data_type comment +pcol int Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -1675,6 +1679,11 @@ age int country string state string +# Partition Information +# col_name data_type comment +country string Transform: identity +state string Transform: identity + # Partition Transform Information # col_name transform_type country IDENTITY @@ -2727,6 +2736,11 @@ age int country string state string +# Partition Information +# col_name data_type comment +country string Transform: identity +state string Transform: identity + # Partition Transform Information # col_name transform_type country IDENTITY @@ -3418,6 +3432,10 @@ datecol date intcol int pcol bigint +# Partition Information +# col_name data_type comment +pcol bigint Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -4108,6 +4126,10 @@ datecol date intcol int pcol double +# Partition Information +# col_name data_type comment +pcol double Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -4798,6 +4820,10 @@ datecol date intcol int pcol decimal(10,6) +# Partition Information +# col_name data_type comment +pcol decimal(10,6) Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out index 1cbaf6d004bf..fcd1c17b24ed 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out @@ -612,6 +612,10 @@ bigintcol bigint intcol int pcol date +# Partition Information +# col_name data_type comment +pcol date Transform: year + # Partition Transform Information # col_name transform_type pcol YEAR @@ -1304,6 +1308,10 @@ bigintcol bigint pcol date intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: month + # Partition Transform Information # col_name transform_type pcol MONTH @@ -1996,6 +2004,10 @@ pcol date bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: day + # Partition Transform Information # col_name transform_type pcol DAY @@ -2447,6 +2459,10 @@ pcol string bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol string Transform: truncate[2] + # Partition Transform Information # col_name transform_type pcol TRUNCATE[2] @@ -2882,6 +2898,10 @@ pcol string bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol string Transform: bucket[16] + # Partition Transform Information # col_name transform_type pcol BUCKET[16] @@ -3105,6 +3125,10 @@ POSTHOOK: Input: default@ice_parquet_decimal_transform_bucket # col_name data_type comment pcol decimal(38,0) +# Partition Information +# col_name data_type comment +pcol decimal(38,0) Transform: bucket[16] + # Partition Transform Information # col_name transform_type pcol BUCKET[16] diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out index 0a724298b074..de49a0d3b366 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out @@ -164,6 +164,10 @@ POSTHOOK: Input: default@testice1000 a int b string +# Partition Information +# col_name data_type comment +b string Transform: truncate[2] + # Partition Transform Information # col_name transform_type b TRUNCATE[2] diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 7aaec8f1924e..063b6389863e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -263,6 +263,10 @@ strcol string intcol int pcol int +# Partition Information +# col_name data_type comment +pcol int Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -1231,6 +1235,11 @@ age int country string state string +# Partition Information +# col_name data_type comment +country string Transform: identity +state string Transform: identity + # Partition Transform Information # col_name transform_type country IDENTITY @@ -1708,6 +1717,10 @@ bigintcol bigint intcol int pcol date +# Partition Information +# col_name data_type comment +pcol date Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -2172,6 +2185,10 @@ datecol date intcol int pcol bigint +# Partition Information +# col_name data_type comment +pcol bigint Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -2636,6 +2653,10 @@ datecol date intcol int pcol double +# Partition Information +# col_name data_type comment +pcol double Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -3100,6 +3121,10 @@ datecol date intcol int pcol decimal(10,6) +# Partition Information +# col_name data_type comment +pcol decimal(10,6) Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out index 7b98a5349057..12745b17f098 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out @@ -608,6 +608,10 @@ bigintcol bigint intcol int pcol date +# Partition Information +# col_name data_type comment +pcol date Transform: year + # Partition Transform Information # col_name transform_type pcol YEAR @@ -1274,6 +1278,10 @@ bigintcol bigint pcol date intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: month + # Partition Transform Information # col_name transform_type pcol MONTH @@ -1944,6 +1952,10 @@ pcol date bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: day + # Partition Transform Information # col_name transform_type pcol DAY @@ -2395,6 +2407,10 @@ pcol string bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol string Transform: truncate[2] + # Partition Transform Information # col_name transform_type pcol TRUNCATE[2] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out index c6a7243ec961..5036ca420f88 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out @@ -92,6 +92,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -170,6 +175,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -256,6 +266,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -340,6 +355,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -410,6 +430,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index b0b02dda8a84..2915bf7d774d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -273,6 +273,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY @@ -865,6 +870,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out index d0195d0d475e..3af63c58b6c2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out @@ -140,6 +140,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -223,6 +227,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out index 4b5738e6b7a9..924c6d9953fa 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out @@ -134,6 +134,10 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -280,6 +284,10 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out index 1f6be0ebf87a..61a931d74a33 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out @@ -228,6 +228,11 @@ event_id int event_time timestamp with local time zone event_src string +# Partition Information +# col_name data_type comment +event_src string Transform: truncate[3] +event_time timestamp with local time zone Transform: month + # Partition Transform Information # col_name transform_type event_src TRUNCATE[3] @@ -334,6 +339,11 @@ event_id int event_time timestamp with local time zone event_src string +# Partition Information +# col_name data_type comment +event_src string Transform: truncate[3] +event_time timestamp with local time zone Transform: month + # Partition Transform Information # col_name transform_type event_src TRUNCATE[3] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out index 96f9e6fe34c1..bc786c1e8dc8 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out @@ -180,6 +180,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY @@ -288,6 +293,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out index ef881a53ba39..5508bdca124c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out @@ -175,6 +175,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -274,6 +278,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -508,6 +516,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -611,6 +623,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out index 9e59529a9e2f..440f6334f114 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out @@ -211,6 +211,10 @@ last_name string dept_id bigint address string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -311,6 +315,10 @@ last_name string dept_id bigint address string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out index ba16b014e3c6..4e120fb8c50d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out @@ -181,6 +181,12 @@ dept_id bigint city string registration_date date +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -287,6 +293,12 @@ dept_id bigint city string registration_date date +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -400,6 +412,12 @@ dept_id bigint city string registration_date date +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out index 23ff2c3bafa8..1d1143f4b635 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out @@ -162,6 +162,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -273,6 +279,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -384,6 +396,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -494,6 +512,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -604,6 +628,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out index f3a5e2964b5a..b01185bb6911 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out @@ -109,6 +109,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY @@ -199,6 +203,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY @@ -303,6 +311,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY @@ -407,6 +419,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out index 2a1860b2b3d5..fa5dcd05e101 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out @@ -68,6 +68,10 @@ id string key int value string +# Partition Information +# col_name data_type comment +key int Transform: bucket[8] + # Partition Transform Information # col_name transform_type key BUCKET[8] @@ -182,6 +186,10 @@ id string key int value string +# Partition Information +# col_name data_type comment +key int Transform: bucket[8] + # Partition Transform Information # col_name transform_type key BUCKET[8] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out index 52ef77680462..62070278505d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out @@ -96,6 +96,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -175,6 +179,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -280,6 +288,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out index e31fb1f787e2..5f31e752db3f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out @@ -881,6 +881,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out index 1902755ec3a1..f1a017215639 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out @@ -610,6 +610,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out index bbd6c589e90a..2feda580b67a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out @@ -513,6 +513,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out index 0bb8f5837f73..7ea7605467ca 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out @@ -50,6 +50,10 @@ POSTHOOK: Input: default@mat1 c int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -138,6 +142,10 @@ POSTHOOK: Input: default@mat2 c int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out index d08fb3ec1095..770cc967d0bc 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out @@ -50,6 +50,11 @@ POSTHOOK: Input: default@mat1 b string c int +# Partition Information +# col_name data_type comment +b string Transform: bucket[16] +c int Transform: truncate[3] + # Partition Transform Information # col_name transform_type b BUCKET[16] @@ -139,6 +144,11 @@ POSTHOOK: Input: default@mat2 b string c int +# Partition Information +# col_name data_type comment +b string Transform: bucket[16] +c int Transform: truncate[3] + # Partition Transform Information # col_name transform_type b BUCKET[16] diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out index 1c4d2e05cdb8..a44d9394025b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out @@ -85,6 +85,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -172,6 +177,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out b/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out index c3c309b656aa..24fff78df880 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out @@ -37,6 +37,11 @@ equality_delete_record_count bigint Count of records in equality d equality_delete_file_count int Count of equality delete files last_updated_at timestamp with local time zone Commit time of snapshot that last updated this partition last_updated_snapshot_id bigint Id of snapshot that last updated this partition + +# Partition Information +# col_name data_type comment +d_part int Transform: identity +e_part int Transform: identity PREHOOK: query: select * from default.ice1.partitions PREHOOK: type: QUERY PREHOOK: Input: default@ice1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out index 0d6f6a02a622..be765c27120a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out @@ -81,6 +81,10 @@ POSTHOOK: Input: default@test_truncate a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -185,6 +189,10 @@ POSTHOOK: Input: default@test_truncate a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out index f37b6a27174a..cd0ce562a725 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out @@ -762,6 +762,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out index 95a19640d864..fdf2679f5b2a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out @@ -531,6 +531,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out index 4f3d0dd65dda..acc7794e12ce 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out @@ -435,6 +435,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java index db1e6a6abbac..cd6f21fbde5f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java @@ -18,9 +18,11 @@ package org.apache.hadoop.hive.ql.ddl.table.info.desc; +import java.util.List; import java.util.Map; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.ddl.DDLWork; @@ -172,7 +174,7 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta if (partitionSpec != null) { Partition part = null; try { - part = db.getPartition(tab, partitionSpec, false); + part = getPartition(tab, partitionSpec); } catch (HiveException e) { // if get exception in finding partition it could be DESCRIBE table key // return null, continue processing for DESCRIBE table key @@ -188,4 +190,16 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta } return null; } + + private Partition getPartition(Table tab, Map partitionSpec) throws HiveException { + boolean isIcebergTable = DDLUtils.isIcebergTable(tab); + if (isIcebergTable) { + List partKeys = tab.getStorageHandler().getPartitionKeys(tab); + if (partKeys.size() != partitionSpec.size()) return null; + List partList = tab.getStorageHandler().getPartitions(tab, partitionSpec, false); + if (partList.isEmpty()) return null; + return partList.getFirst(); + } + return db.getPartition(tab, partitionSpec, false); + } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index 9f9f9351ef48..fb3d81db8ffa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.ShowUtils; import org.apache.hadoop.hive.ql.ddl.table.info.desc.formatter.DescTableFormatter; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -127,7 +128,14 @@ private Table getTable() throws HiveException { private Partition getPartition(Table table) throws HiveException { Partition part = null; if (desc.getPartitionSpec() != null) { - part = context.getDb().getPartition(table, desc.getPartitionSpec(), false); + boolean isIcebergTable = DDLUtils.isIcebergTable(table); + if (isIcebergTable) { + List partList = table.getStorageHandler().getPartitions(table, desc.getPartitionSpec(), false); + part = (partList.isEmpty()) ? null : partList.getFirst(); + + } else { + part = context.getDb().getPartition(table, desc.getPartitionSpec(), false); + } if (part == null) { throw new HiveException(ErrorMsg.INVALID_PARTITION, StringUtils.join(desc.getPartitionSpec().keySet(), ','), desc.getDbTableName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 6f73c36b6ad1..025bc09600a9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.metastore.api.SourceTable; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.ShowUtils; import org.apache.hadoop.hive.ql.ddl.ShowUtils.TextMetaDataTable; import org.apache.hadoop.hive.ql.ddl.table.info.desc.DescTableDesc; @@ -171,7 +172,10 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column boolean isFormatted, boolean isOutputPadded) throws IOException { String partitionData = ""; if (columnPath == null) { - List partitionColumns = table.isPartitioned() ? table.getPartCols() : null; + boolean isIcebergTable = DDLUtils.isIcebergTable(table); + List partitionColumns = table.isPartitioned() + ? (isIcebergTable ? table.getStorageHandler().getPartitionKeys(table) : table.getPartCols()) + : null; if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { TextMetaDataTable metaDataTable = new TextMetaDataTable(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 89681572c952..d150f90f8506 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -56,6 +56,7 @@ public DummyPartition(Table tbl, String name, Map partSpec) { new org.apache.hadoop.hive.metastore.api.Partition(); tPart.setSd(tbl.getSd().deepCopy()); tPart.setParameters(Maps.newHashMap()); + tPart.setDbName(tbl.getDbName()); this.partSpec = Maps.newLinkedHashMap(partSpec); setTPartition(tPart); @@ -80,8 +81,9 @@ public LinkedHashMap getSpec() { @Override public List getValues() { + Table table = this.getTable(); List values = new ArrayList(); - for (FieldSchema fs : this.getTable().getPartCols()) { + for (FieldSchema fs : table.getStorageHandler().getPartitionKeys(table)) { values.add(partSpec.get(fs.getName())); } return values; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 180431206109..7c2c771596a2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -68,6 +68,7 @@ import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.cache.results.CacheUsage; import org.apache.hadoop.hive.ql.ddl.DDLDesc.DDLDescWithWriteId; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.Task; @@ -1759,7 +1760,7 @@ public static void validatePartSpec(Table tbl, Map partSpec, * @param partitionClausePresent Whether a partition clause is present in the query (e.g. PARTITION(last_name='Don')) */ protected static void validateUnsupportedPartitionClause(Table tbl, boolean partitionClausePresent) { - if (partitionClausePresent && tbl.hasNonNativePartitionSupport()) { + if (partitionClausePresent && tbl.hasNonNativePartitionSupport() && !DDLUtils.isIcebergTable(tbl)) { throw new UnsupportedOperationException("Using partition spec in query is unsupported for non-native table" + " backed by: " + tbl.getStorageHandler().toString()); } From ea8d0621aac70d2dc38e9c13ffd3c98efadf47f8 Mon Sep 17 00:00:00 2001 From: ragupta Date: Fri, 9 Jan 2026 10:31:47 +0530 Subject: [PATCH 02/19] addressed sonar issues --- .../hive/ql/ddl/table/info/desc/DescTableAnalyzer.java | 8 ++++++-- .../table/info/desc/formatter/TextDescTableFormatter.java | 7 ++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java index cd6f21fbde5f..1743cc2d81bb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java @@ -195,9 +195,13 @@ private Partition getPartition(Table tab, Map partitionSpec) thr boolean isIcebergTable = DDLUtils.isIcebergTable(tab); if (isIcebergTable) { List partKeys = tab.getStorageHandler().getPartitionKeys(tab); - if (partKeys.size() != partitionSpec.size()) return null; + if (partKeys.size() != partitionSpec.size()) { + return null; + } List partList = tab.getStorageHandler().getPartitions(tab, partitionSpec, false); - if (partList.isEmpty()) return null; + if (partList.isEmpty()) { + return null; + } return partList.getFirst(); } return db.getPartition(tab, partitionSpec, false); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 025bc09600a9..6b12a34cf14d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -173,9 +173,10 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column String partitionData = ""; if (columnPath == null) { boolean isIcebergTable = DDLUtils.isIcebergTable(table); - List partitionColumns = table.isPartitioned() - ? (isIcebergTable ? table.getStorageHandler().getPartitionKeys(table) : table.getPartCols()) - : null; + List partitionColumns = null; + if (table.isPartitioned()) { + partitionColumns = isIcebergTable ? table.getStorageHandler().getPartitionKeys(table) : table.getPartCols(); + } if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { TextMetaDataTable metaDataTable = new TextMetaDataTable(); From 3b6f24b58baf1c42088aa67898f1363052caeb45 Mon Sep 17 00:00:00 2001 From: ragupta Date: Wed, 14 Jan 2026 13:18:20 +0530 Subject: [PATCH 03/19] separated redundant code and made implementation generic to non native tables --- .../mr/hive/HiveIcebergStorageHandler.java | 7 ++++ .../table/info/desc/DescTableAnalyzer.java | 37 +++++-------------- .../table/info/desc/DescTableOperation.java | 10 +---- .../formatter/TextDescTableFormatter.java | 6 +-- .../hive/ql/metadata/HiveStorageHandler.java | 5 +++ .../hive/ql/parse/BaseSemanticAnalyzer.java | 18 --------- 6 files changed, 25 insertions(+), 58 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 2978341b6da5..0dbf20003ff4 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2075,6 +2075,13 @@ public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) return table.spec().isPartitioned(); } + @Override + public boolean isPartitionPresent(org.apache.hadoop.hive.ql.metadata.Table table, + Map partitionSpec) throws SemanticException { + return getPartitionKeys(table).size() == partitionSpec.size() && + !getPartitions(table, partitionSpec, false).isEmpty(); + } + @Override public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec, RewritePolicy policy) throws SemanticException { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java index 1743cc2d81bb..fdc9176c6d93 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java @@ -18,11 +18,9 @@ package org.apache.hadoop.hive.ql.ddl.table.info.desc; -import java.util.List; import java.util.Map; import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.ddl.DDLWork; @@ -35,7 +33,6 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; -import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; @@ -151,20 +148,18 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta if (node.getChild(1).getType() == HiveParser.TOK_PARTSPEC) { ASTNode partNode = (ASTNode) node.getChild(1); - Table tab = null; + Table table; try { - tab = db.getTable(tableName.getNotEmptyDbTable()); + table = db.getTable(tableName.getNotEmptyDbTable()); } catch (InvalidTableException e) { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName.getNotEmptyDbTable()), e); } catch (HiveException e) { throw new SemanticException(e.getMessage(), e); } - Map partitionSpec = null; + Map partitionSpec; try { - partitionSpec = getPartSpec(partNode); - validateUnsupportedPartitionClause(tab, partitionSpec != null && !partitionSpec.isEmpty()); - partitionSpec = getValidatedPartSpec(tab, partNode, db.getConf(), false); + partitionSpec = getValidatedPartSpec(table, partNode, db.getConf(), false); } catch (SemanticException e) { // get exception in resolving partition it could be DESCRIBE table key // return null, continue processing for DESCRIBE table key @@ -172,16 +167,18 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta } if (partitionSpec != null) { - Partition part = null; + boolean isPartitionPresent; try { - part = getPartition(tab, partitionSpec); + isPartitionPresent = table.isNonNative() ? + table.getStorageHandler().isPartitionPresent(table, partitionSpec) : + db.getPartition(table, partitionSpec) != null; } catch (HiveException e) { // if get exception in finding partition it could be DESCRIBE table key // return null, continue processing for DESCRIBE table key return null; } - if (part == null) { + if (!isPartitionPresent) { throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partitionSpec.toString())); } @@ -190,20 +187,4 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta } return null; } - - private Partition getPartition(Table tab, Map partitionSpec) throws HiveException { - boolean isIcebergTable = DDLUtils.isIcebergTable(tab); - if (isIcebergTable) { - List partKeys = tab.getStorageHandler().getPartitionKeys(tab); - if (partKeys.size() != partitionSpec.size()) { - return null; - } - List partList = tab.getStorageHandler().getPartitions(tab, partitionSpec, false); - if (partList.isEmpty()) { - return null; - } - return partList.getFirst(); - } - return db.getPartition(tab, partitionSpec, false); - } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index fb3d81db8ffa..6fef8f953ac4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; -import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.ShowUtils; import org.apache.hadoop.hive.ql.ddl.table.info.desc.formatter.DescTableFormatter; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -128,14 +127,7 @@ private Table getTable() throws HiveException { private Partition getPartition(Table table) throws HiveException { Partition part = null; if (desc.getPartitionSpec() != null) { - boolean isIcebergTable = DDLUtils.isIcebergTable(table); - if (isIcebergTable) { - List partList = table.getStorageHandler().getPartitions(table, desc.getPartitionSpec(), false); - part = (partList.isEmpty()) ? null : partList.getFirst(); - - } else { - part = context.getDb().getPartition(table, desc.getPartitionSpec(), false); - } + part = context.getDb().getPartition(table, desc.getPartitionSpec()); if (part == null) { throw new HiveException(ErrorMsg.INVALID_PARTITION, StringUtils.join(desc.getPartitionSpec().keySet(), ','), desc.getDbTableName()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 6b12a34cf14d..85c1a3928566 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hive.metastore.api.SourceTable; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.ShowUtils; import org.apache.hadoop.hive.ql.ddl.ShowUtils.TextMetaDataTable; import org.apache.hadoop.hive.ql.ddl.table.info.desc.DescTableDesc; @@ -172,10 +171,11 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column boolean isFormatted, boolean isOutputPadded) throws IOException { String partitionData = ""; if (columnPath == null) { - boolean isIcebergTable = DDLUtils.isIcebergTable(table); List partitionColumns = null; if (table.isPartitioned()) { - partitionColumns = isIcebergTable ? table.getStorageHandler().getPartitionKeys(table) : table.getPartCols(); + partitionColumns = table.isNonNative() ? + table.getStorageHandler().getPartitionKeys(table) : + table.getPartCols(); } if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 39358ca3d594..6cf49f163943 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -987,6 +987,11 @@ default boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table table) { return false; } + default boolean isPartitionPresent(org.apache.hadoop.hive.ql.metadata.Table table, + Map partitionSpec) throws SemanticException { + return false; + } + default boolean hasUndergonePartitionEvolution(org.apache.hadoop.hive.ql.metadata.Table table) { throw new UnsupportedOperationException("Storage handler does not support checking if table " + "undergone partition evolution."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 7c2c771596a2..a0a0f7500806 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -68,7 +68,6 @@ import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.cache.results.CacheUsage; import org.apache.hadoop.hive.ql.ddl.DDLDesc.DDLDescWithWriteId; -import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.Task; @@ -1749,23 +1748,6 @@ public static void validatePartSpec(Table tbl, Map partSpec, } } - /** - * Throws an UnsupportedOperationException in case the query has a partition clause but the table is never partitioned - * on the HMS-level. Even though table is not partitioned from the HMS's point of view, it might have some other - * notion of partitioning under the hood (e.g. Iceberg tables). In these cases, we might decide to proactively throw a - * more descriptive, unified error message instead of failing on some other semantic analysis validation step, which - * could provide a more counter-intuitive exception message. - * - * @param tbl The table object, should not be null. - * @param partitionClausePresent Whether a partition clause is present in the query (e.g. PARTITION(last_name='Don')) - */ - protected static void validateUnsupportedPartitionClause(Table tbl, boolean partitionClausePresent) { - if (partitionClausePresent && tbl.hasNonNativePartitionSupport() && !DDLUtils.isIcebergTable(tbl)) { - throw new UnsupportedOperationException("Using partition spec in query is unsupported for non-native table" + - " backed by: " + tbl.getStorageHandler().toString()); - } - } - public static void validatePartColumnType(Table tbl, Map partSpec, ASTNode astNode, HiveConf conf) throws SemanticException { if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TYPE_CHECK_ON_INSERT)) { From 039659a052c558dace180f9f2e26e7a58188b378 Mon Sep 17 00:00:00 2001 From: ragupta Date: Fri, 16 Jan 2026 14:00:31 +0530 Subject: [PATCH 04/19] used hasNonNativePartitionSupport() in place of isNonNative() --- .../table/info/desc/formatter/TextDescTableFormatter.java | 2 +- .../org/apache/hadoop/hive/ql/metadata/DummyPartition.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 85c1a3928566..2847f182a8a1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -173,7 +173,7 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column if (columnPath == null) { List partitionColumns = null; if (table.isPartitioned()) { - partitionColumns = table.isNonNative() ? + partitionColumns = table.hasNonNativePartitionSupport() ? table.getStorageHandler().getPartitionKeys(table) : table.getPartCols(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index d150f90f8506..1e488221e44a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -83,7 +83,10 @@ public LinkedHashMap getSpec() { public List getValues() { Table table = this.getTable(); List values = new ArrayList(); - for (FieldSchema fs : table.getStorageHandler().getPartitionKeys(table)) { + for (FieldSchema fs : + table.hasNonNativePartitionSupport() ? + table.getStorageHandler().getPartitionKeys(table) : + table.getPartCols()) { values.add(partSpec.get(fs.getName())); } return values; From 786e3de27c8bb213b0c9b0061add41091c6bf85d Mon Sep 17 00:00:00 2001 From: ragupta Date: Fri, 16 Jan 2026 17:30:57 +0530 Subject: [PATCH 05/19] updated getPartition Api --- .../mr/hive/HiveIcebergStorageHandler.java | 23 +++- .../queries/positive/desc_ice_tbl_part_spec.q | 15 ++- .../positive/desc_ice_tbl_part_spec.q.out | 107 ++++++++++++++++-- .../table/info/desc/DescTableAnalyzer.java | 9 +- .../hive/ql/metadata/HiveStorageHandler.java | 5 - .../hive/ql/parse/BaseSemanticAnalyzer.java | 12 +- 6 files changed, 140 insertions(+), 31 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 0dbf20003ff4..b21e7c8f55fe 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2075,11 +2075,21 @@ public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) return table.spec().isPartitioned(); } - @Override - public boolean isPartitionPresent(org.apache.hadoop.hive.ql.metadata.Table table, - Map partitionSpec) throws SemanticException { - return getPartitionKeys(table).size() == partitionSpec.size() && - !getPartitions(table, partitionSpec, false).isEmpty(); + private boolean isPartitionPresent(org.apache.hadoop.hive.ql.metadata.Table table, + Map partitionSpec) throws SemanticException { + try { + List partNames = getPartitionNames(table, partitionSpec); + for (String partName : partNames) { + if (Warehouse.makeSpecFromName(partName).size() == partitionSpec.size()) { + return true; + } + } + } catch (HiveException e) { + return false; + } catch (MetaException e) { + throw new SemanticException("Unable to construct spec for partition name due to: ", e); + } + return false; } @Override @@ -2087,6 +2097,9 @@ public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec, RewritePolicy policy) throws SemanticException { validatePartSpec(table, partitionSpec, policy); try { + if (!isPartitionPresent(table, partitionSpec)) { + return null; + } String partName = Warehouse.makePartName(partitionSpec, false); return new DummyPartition(table, partName, partitionSpec); } catch (MetaException e) { diff --git a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q index c020c08997a1..1246cecec7ed 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q @@ -1,9 +1,14 @@ drop table if exists ice_t; create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; -insert into table ice_t values( 1, "hello1" ,2, "hello2"); -insert into table ice_t values( 3, "hello3" ,4, "hello4"); -insert into table ice_t values( 5, "hello5" ,6, "hello6"); +insert into table ice_t values( 1, "hello1" ,2, "hello2" ); +insert into table ice_t values( 3, "hello3" ,4, "hello4" ); +insert into table ice_t values( 5, "hello5" ,6, "hello6" ); -desc extended ice_t PARTITION(c=6,d="hello6"); -desc formatted ice_t PARTITION(c=6,d="hello6"); \ No newline at end of file +desc extended ice_t PARTITION( c=6, d="hello6" ); +desc formatted ice_t PARTITION( c=6, d="hello6" ); + +alter table ice_t set partition spec ( c, d, b ); +insert into table ice_t values( 7, "hello7" , 8 , "hello8" ); +desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ); +desc formatted ice_t PARTITION( c=4 , d="hello4" ); diff --git a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out index f62c7309ecf2..92f587f9b519 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out @@ -12,34 +12,34 @@ POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@ice_t -PREHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2") +PREHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2" ) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_t -POSTHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2") +POSTHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2" ) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t -PREHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4") +PREHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4" ) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_t -POSTHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4") +POSTHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4" ) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t -PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6" ) PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_t -POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6" ) POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t -PREHOOK: query: desc extended ice_t PARTITION(c=6,d="hello6") +PREHOOK: query: desc extended ice_t PARTITION( c=6, d="hello6" ) PREHOOK: type: DESCTABLE PREHOOK: Input: default@ice_t -POSTHOOK: query: desc extended ice_t PARTITION(c=6,d="hello6") +POSTHOOK: query: desc extended ice_t PARTITION( c=6, d="hello6" ) POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@ice_t a int @@ -58,10 +58,10 @@ c IDENTITY d IDENTITY #### A masked pattern was here #### -PREHOOK: query: desc formatted ice_t PARTITION(c=6,d="hello6") +PREHOOK: query: desc formatted ice_t PARTITION( c=6, d="hello6" ) PREHOOK: type: DESCTABLE PREHOOK: Input: default@ice_t -POSTHOOK: query: desc formatted ice_t PARTITION(c=6,d="hello6") +POSTHOOK: query: desc formatted ice_t PARTITION( c=6, d="hello6" ) POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@ice_t # col_name data_type comment @@ -92,3 +92,90 @@ InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat Compressed: No Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] +PREHOOK: query: alter table ice_t set partition spec ( c, d, b ) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_t +POSTHOOK: query: alter table ice_t set partition spec ( c, d, b ) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: insert into table ice_t values( 7, "hello7" , 8 , "hello8" ) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: insert into table ice_t values( 7, "hello7" , 8 , "hello8" ) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity +b string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY +b IDENTITY + +# Detailed Partition Information +Partition Value: [8, hello8, hello7] +Database: default +Table: ice_t +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] +PREHOOK: query: desc formatted ice_t PARTITION( c=4 , d="hello4" ) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: desc formatted ice_t PARTITION( c=4 , d="hello4" ) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity +b string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY +b IDENTITY + +# Detailed Partition Information +Partition Value: [4, hello4, null] +Database: default +Table: ice_t +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java index fdc9176c6d93..53edd29ce0be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; +import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; @@ -167,18 +168,16 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta } if (partitionSpec != null) { - boolean isPartitionPresent; + Partition part; try { - isPartitionPresent = table.isNonNative() ? - table.getStorageHandler().isPartitionPresent(table, partitionSpec) : - db.getPartition(table, partitionSpec) != null; + part = db.getPartition(table, partitionSpec); } catch (HiveException e) { // if get exception in finding partition it could be DESCRIBE table key // return null, continue processing for DESCRIBE table key return null; } - if (!isPartitionPresent) { + if (part == null) { throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partitionSpec.toString())); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 6cf49f163943..39358ca3d594 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -987,11 +987,6 @@ default boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table table) { return false; } - default boolean isPartitionPresent(org.apache.hadoop.hive.ql.metadata.Table table, - Map partitionSpec) throws SemanticException { - return false; - } - default boolean hasUndergonePartitionEvolution(org.apache.hadoop.hive.ql.metadata.Table table) { throw new UnsupportedOperationException("Storage handler does not support checking if table " + "undergone partition evolution."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index a0a0f7500806..4d85ff596b42 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.DataConnector; import org.apache.hadoop.hive.metastore.api.Database; @@ -59,6 +60,7 @@ import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.SourceTable; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.Context; @@ -87,6 +89,7 @@ import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; @@ -1354,7 +1357,14 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartit if (partHandle == null) { // if partSpec doesn't exists in DB, return a delegate one // and the actual partition is created in MoveTask - partHandle = new Partition(tableHandle, partSpec, null); + try { + partHandle = tableHandle.hasNonNativePartitionSupport() ? + new DummyPartition(tableHandle, Warehouse.makePartName(partSpec, false), partSpec) : + new Partition(tableHandle, partSpec, null); + } catch (MetaException e) { + throw new SemanticException("Unable to construct name for dummy partition due to: ", e); + } + } else { partitions.add(partHandle); } From aa3f65457920ab042de8514741fb847b5521f422 Mon Sep 17 00:00:00 2001 From: ragupta Date: Sun, 18 Jan 2026 15:35:48 +0530 Subject: [PATCH 06/19] moved partition presence method --- .../mr/hive/HiveIcebergStorageHandler.java | 20 ++----------------- .../iceberg/mr/hive/IcebergTableUtil.java | 11 ++++++++++ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index b21e7c8f55fe..c3e0e2796768 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2075,29 +2075,13 @@ public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) return table.spec().isPartitioned(); } - private boolean isPartitionPresent(org.apache.hadoop.hive.ql.metadata.Table table, - Map partitionSpec) throws SemanticException { - try { - List partNames = getPartitionNames(table, partitionSpec); - for (String partName : partNames) { - if (Warehouse.makeSpecFromName(partName).size() == partitionSpec.size()) { - return true; - } - } - } catch (HiveException e) { - return false; - } catch (MetaException e) { - throw new SemanticException("Unable to construct spec for partition name due to: ", e); - } - return false; - } - @Override public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec, RewritePolicy policy) throws SemanticException { validatePartSpec(table, partitionSpec, policy); try { - if (!isPartitionPresent(table, partitionSpec)) { + Table icebergTable = IcebergTableUtil.getTable(conf, table.getTTable()); + if (!IcebergTableUtil.hasPartition(icebergTable, partitionSpec)) { return null; } String partName = Warehouse.makePartName(partitionSpec, false); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 6a6b83f6cc15..dbaf7b826a66 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -567,6 +567,17 @@ public static List getPartitionNames(Table table, Map pa } } + public static boolean hasPartition(Table icebergTable, + Map partitionSpec) throws MetaException { + try { + List partNames = getPartitionNames(icebergTable, partitionSpec, false); + return !partNames.isEmpty() && + Warehouse.makePartName(partitionSpec, false).equals(partNames.getFirst()); + } catch (HiveException e) { + return false; + } + } + public static PartitionSpec getPartitionSpec(Table icebergTable, String partitionPath) throws MetaException, HiveException { if (icebergTable == null || partitionPath == null || partitionPath.isEmpty()) { From 863d29efb8debfedd81586a8ff22028e4dc218b0 Mon Sep 17 00:00:00 2001 From: ragupta Date: Tue, 20 Jan 2026 20:16:49 +0530 Subject: [PATCH 07/19] refractor methods --- .../mr/hive/HiveIcebergStorageHandler.java | 11 +---------- .../iceberg/mr/hive/IcebergTableUtil.java | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index c3e0e2796768..84b750c218f2 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2079,16 +2079,7 @@ public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec, RewritePolicy policy) throws SemanticException { validatePartSpec(table, partitionSpec, policy); - try { - Table icebergTable = IcebergTableUtil.getTable(conf, table.getTTable()); - if (!IcebergTableUtil.hasPartition(icebergTable, partitionSpec)) { - return null; - } - String partName = Warehouse.makePartName(partitionSpec, false); - return new DummyPartition(table, partName, partitionSpec); - } catch (MetaException e) { - throw new SemanticException("Unable to construct name for dummy partition due to: ", e); - } + return IcebergTableUtil.getPartition(conf, table, partitionSpec); } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index dbaf7b826a66..24b1dcd314c2 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -649,13 +649,27 @@ public static boolean hasUndergonePartitionEvolution(Table table) { .anyMatch(id -> id != table.spec().specId()); } + public static Partition getPartition(Configuration conf, + org.apache.hadoop.hive.ql.metadata.Table table, + Map partitionSpec) throws SemanticException { + Table icebergTable = IcebergTableUtil.getTable(conf, table.getTTable()); + try { + if (!IcebergTableUtil.hasPartition(icebergTable, partitionSpec)) { + return null; + } + String partName = Warehouse.makePartName(partitionSpec, false); + return new DummyPartition(table, partName, partitionSpec); + } catch (MetaException e) { + throw new SemanticException("Unable to construct name for dummy partition due to: ", e); + } + } + public static > Set getPartitionNames(Table icebergTable, Iterable files, Boolean latestSpecOnly) { Set partitions = Sets.newHashSet(); int tableSpecId = icebergTable.spec().specId(); for (T file : files) { - if (latestSpecOnly == null || Boolean.TRUE.equals(latestSpecOnly) && file.specId() == tableSpecId || - Boolean.FALSE.equals(latestSpecOnly) && file.specId() != tableSpecId) { + if (latestSpecOnly == null || latestSpecOnly.equals(file.specId() == tableSpecId)) { String partName = icebergTable.specs().get(file.specId()).partitionToPath(file.partition()); partitions.add(partName); } From 49d8454994b8a9a553d04700c1409db507a15ea3 Mon Sep 17 00:00:00 2001 From: ragupta Date: Wed, 21 Jan 2026 10:03:28 +0530 Subject: [PATCH 08/19] dropped hasPartition --- .../mr/hive/HiveIcebergStorageHandler.java | 3 +- .../iceberg/mr/hive/IcebergTableUtil.java | 34 ++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 84b750c218f2..769b6b79c979 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2091,8 +2091,7 @@ public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table, */ public List getPartitionNames(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Map partitionSpec) throws SemanticException { - Table icebergTable = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - return IcebergTableUtil.getPartitionNames(icebergTable, partitionSpec, false); + return IcebergTableUtil.getPartitionNames(conf, hmsTable, partitionSpec, false); } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 24b1dcd314c2..cd56f6419671 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -567,17 +567,6 @@ public static List getPartitionNames(Table table, Map pa } } - public static boolean hasPartition(Table icebergTable, - Map partitionSpec) throws MetaException { - try { - List partNames = getPartitionNames(icebergTable, partitionSpec, false); - return !partNames.isEmpty() && - Warehouse.makePartName(partitionSpec, false).equals(partNames.getFirst()); - } catch (HiveException e) { - return false; - } - } - public static PartitionSpec getPartitionSpec(Table icebergTable, String partitionPath) throws MetaException, HiveException { if (icebergTable == null || partitionPath == null || partitionPath.isEmpty()) { @@ -652,18 +641,31 @@ public static boolean hasUndergonePartitionEvolution(Table table) { public static Partition getPartition(Configuration conf, org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec) throws SemanticException { - Table icebergTable = IcebergTableUtil.getTable(conf, table.getTTable()); try { - if (!IcebergTableUtil.hasPartition(icebergTable, partitionSpec)) { + List partNames = getPartitionNames(conf, table, partitionSpec, false); + if (partNames.isEmpty()) { return null; } - String partName = Warehouse.makePartName(partitionSpec, false); - return new DummyPartition(table, partName, partitionSpec); + String expectedName = Warehouse.makePartName(partitionSpec, false); + if (!expectedName.equals(partNames.getFirst())) { + return null; + } + return new DummyPartition(table, expectedName, partitionSpec); + } catch (SemanticException e) { + return null; } catch (MetaException e) { - throw new SemanticException("Unable to construct name for dummy partition due to: ", e); + throw new SemanticException("Unable to construct dummy partition", e); } } + public static List getPartitionNames(Configuration conf, + org.apache.hadoop.hive.ql.metadata.Table table, + Map partitionSpec, + boolean latestSpecOnly) throws SemanticException { + Table icebergTable = getTable(conf, table.getTTable()); + return getPartitionNames(icebergTable, partitionSpec, latestSpecOnly); + } + public static > Set getPartitionNames(Table icebergTable, Iterable files, Boolean latestSpecOnly) { Set partitions = Sets.newHashSet(); From 6bc21a852824f3650bef3bbb26e1a6b492b94e8e Mon Sep 17 00:00:00 2001 From: ragupta Date: Wed, 21 Jan 2026 20:52:06 +0530 Subject: [PATCH 09/19] normalised methods --- .../mr/hive/HiveIcebergStorageHandler.java | 3 +- .../iceberg/mr/hive/IcebergTableUtil.java | 66 +++++++++---------- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 769b6b79c979..8bb3d2a23ca3 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2056,8 +2056,7 @@ public boolean canUseTruncate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, @Override public List getPartitions(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Map partitionSpec, boolean latestSpecOnly) throws SemanticException { - Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - List partNames = IcebergTableUtil.getPartitionNames(table, partitionSpec, latestSpecOnly); + List partNames = IcebergTableUtil.getPartitionNames(conf, hmsTable, partitionSpec, latestSpecOnly); return IcebergTableUtil.convertNameToMetastorePartition(hmsTable, partNames); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index cd56f6419671..e7adf9baffd9 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -527,6 +527,27 @@ public static List getPartitionFields(Table table, boolean lates .collect(Collectors.toList()); } + public static Partition getPartition(Configuration conf, + org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec) + throws SemanticException { + List partNames = + getPartitionNames(conf, table, partitionSpec, false); + + if (partNames.isEmpty()) { + return null; + } + + try { + String expectedName = Warehouse.makePartName(partitionSpec, false); + if (!expectedName.equals(partNames.getFirst())) { + return null; + } + return new DummyPartition(table, expectedName, partitionSpec); + } catch (MetaException e) { + throw new SemanticException("Unable to construct dummy partition", e); + } + } + /** * Returns a list of partition names satisfying the provided partition spec. * @param table Iceberg table @@ -534,29 +555,31 @@ public static List getPartitionFields(Table table, boolean lates * @param latestSpecOnly when True, returns partitions with the current spec only, else - any specs * @return List of partition names */ - public static List getPartitionNames(Table table, Map partSpecMap, + public static List getPartitionNames(Configuration conf, + org.apache.hadoop.hive.ql.metadata.Table table, Map partSpecMap, boolean latestSpecOnly) throws SemanticException { + Table icebergTable = getTable(conf, table.getTTable()); Expression expression = IcebergTableUtil.generateExpressionFromPartitionSpec( - table, partSpecMap, latestSpecOnly); + icebergTable, partSpecMap, latestSpecOnly); PartitionsTable partitionsTable = (PartitionsTable) MetadataTableUtils.createMetadataTableInstance( - table, MetadataTableType.PARTITIONS); + icebergTable, MetadataTableType.PARTITIONS); try (CloseableIterable fileScanTasks = partitionsTable.newScan().planFiles()) { return FluentIterable.from(fileScanTasks) .transformAndConcat(task -> task.asDataTask().rows()) .transform(row -> { StructLike data = row.get(IcebergTableUtil.PART_IDX, StructProjection.class); - PartitionSpec spec = table.specs().get(row.get(IcebergTableUtil.SPEC_IDX, Integer.class)); + PartitionSpec spec = icebergTable.specs().get(row.get(IcebergTableUtil.SPEC_IDX, Integer.class)); return Maps.immutableEntry( IcebergTableUtil.toPartitionData( - data, Partitioning.partitionType(table), spec.partitionType()), + data, Partitioning.partitionType(icebergTable), spec.partitionType()), spec); }).filter(e -> { ResidualEvaluator resEval = ResidualEvaluator.of(e.getValue(), expression, false); return e.getValue().isPartitioned() && resEval.residualFor(e.getKey()).isEquivalentTo(Expressions.alwaysTrue()) && - (e.getValue().specId() == table.spec().specId() || !latestSpecOnly); + (e.getValue().specId() == icebergTable.spec().specId() || !latestSpecOnly); }).transform(e -> e.getValue().partitionToPath(e.getKey())).toSortedList( Comparator.naturalOrder()); @@ -638,40 +661,13 @@ public static boolean hasUndergonePartitionEvolution(Table table) { .anyMatch(id -> id != table.spec().specId()); } - public static Partition getPartition(Configuration conf, - org.apache.hadoop.hive.ql.metadata.Table table, - Map partitionSpec) throws SemanticException { - try { - List partNames = getPartitionNames(conf, table, partitionSpec, false); - if (partNames.isEmpty()) { - return null; - } - String expectedName = Warehouse.makePartName(partitionSpec, false); - if (!expectedName.equals(partNames.getFirst())) { - return null; - } - return new DummyPartition(table, expectedName, partitionSpec); - } catch (SemanticException e) { - return null; - } catch (MetaException e) { - throw new SemanticException("Unable to construct dummy partition", e); - } - } - - public static List getPartitionNames(Configuration conf, - org.apache.hadoop.hive.ql.metadata.Table table, - Map partitionSpec, - boolean latestSpecOnly) throws SemanticException { - Table icebergTable = getTable(conf, table.getTTable()); - return getPartitionNames(icebergTable, partitionSpec, latestSpecOnly); - } - public static > Set getPartitionNames(Table icebergTable, Iterable files, Boolean latestSpecOnly) { Set partitions = Sets.newHashSet(); int tableSpecId = icebergTable.spec().specId(); for (T file : files) { - if (latestSpecOnly == null || latestSpecOnly.equals(file.specId() == tableSpecId)) { + if (latestSpecOnly == null || latestSpecOnly && file.specId() == tableSpecId || + !latestSpecOnly && file.specId() != tableSpecId) { String partName = icebergTable.specs().get(file.specId()).partitionToPath(file.partition()); partitions.add(partName); } From 0d9ed7095f2879994565bd6fed292728b4a333c1 Mon Sep 17 00:00:00 2001 From: ragupta Date: Thu, 22 Jan 2026 02:25:50 +0530 Subject: [PATCH 10/19] fix transform based partition spec scenario --- .../mr/hive/HiveIcebergFilterFactory.java | 29 +---- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 5 +- .../mr/hive/HiveIcebergStorageHandler.java | 11 +- .../iceberg/mr/hive/IcebergTableUtil.java | 106 ++++++------------ .../apache/iceberg/mr/hive/SchemaUtils.java | 77 +++++++++++++ .../mr/hive/TestHiveIcebergPartitions.java | 20 ++++ .../queries/positive/desc_ice_tbl_part_spec.q | 2 +- .../positive/desc_ice_tbl_part_spec.q.out | 12 +- 8 files changed, 144 insertions(+), 118 deletions(-) create mode 100644 iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergFilterFactory.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergFilterFactory.java index d1d98b3a584e..cc15053bd7a5 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergFilterFactory.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergFilterFactory.java @@ -30,7 +30,6 @@ import java.time.temporal.ChronoUnit; import java.util.List; import java.util.stream.Collectors; -import org.apache.commons.lang3.ObjectUtils; import org.apache.hadoop.hive.ql.io.sarg.ExpressionTree; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; @@ -111,9 +110,7 @@ private static Expression translate(ExpressionTree tree, List lea */ private static Expression translateLeaf(PredicateLeaf leaf) { TransformSpec transformSpec = TransformSpec.fromStringWithColumnName(leaf.getColumnName()); - String columnName = transformSpec.getColumnName(); - UnboundTerm column = - ObjectUtils.defaultIfNull(toTerm(columnName, transformSpec), Expressions.ref(columnName)); + UnboundTerm column = SchemaUtils.toTerm(transformSpec); switch (leaf.getOperator()) { case EQUALS: @@ -144,30 +141,6 @@ private static Expression translateLeaf(PredicateLeaf leaf) { } } - public static UnboundTerm toTerm(String columnName, TransformSpec transformSpec) { - if (transformSpec == null) { - return null; - } - switch (transformSpec.getTransformType()) { - case YEAR: - return Expressions.year(columnName); - case MONTH: - return Expressions.month(columnName); - case DAY: - return Expressions.day(columnName); - case HOUR: - return Expressions.hour(columnName); - case TRUNCATE: - return Expressions.truncate(columnName, transformSpec.getTransformParam()); - case BUCKET: - return Expressions.bucket(columnName, transformSpec.getTransformParam()); - case IDENTITY: - return null; - default: - throw new UnsupportedOperationException("Unknown transformSpec: " + transformSpec); - } - } - // PredicateLeafImpl has a work-around for Kryo serialization with java.util.Date objects where it converts values to // Timestamp using Date#getTime. This conversion discards microseconds, so this is a necessary to avoid it. private static final DynFields.UnboundField LITERAL_FIELD = DynFields.builder() diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 64f81d35a902..b87d56066a3a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -35,7 +35,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ObjectUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -1029,9 +1028,7 @@ private static UnboundPredicate getPartitionPredicate(PartitionData part String columName = schema.findField(field.sourceId()).name(); TransformSpec transformSpec = TransformSpec.fromString(field.transform().toString(), columName); - UnboundTerm partitionColumn = - ObjectUtils.defaultIfNull(HiveIcebergFilterFactory.toTerm(columName, transformSpec), - Expressions.ref(field.name())); + UnboundTerm partitionColumn = SchemaUtils.toTerm(transformSpec); return Expressions.equal(partitionColumn, partitionData.get(index, Object.class)); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 8bb3d2a23ca3..362c35d7d834 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -866,8 +866,7 @@ public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.m return table.spec().fields().stream() .filter(f -> !f.transform().isVoid()) .map(f -> { - TransformSpec spec = IcebergTableUtil.getTransformSpec( - table, f.transform().toString(), f.sourceId()); + TransformSpec spec = IcebergTableUtil.getTransformSpec(table, f.transform().toString(), f.sourceId()); spec.setFieldName(f.name()); return spec; }) @@ -882,8 +881,7 @@ public Map> getPartitionTransformSpecs( e.getValue().fields().stream() .filter(f -> !f.transform().isVoid()) .map(f -> { - TransformSpec spec = IcebergTableUtil.getTransformSpec( - table, f.transform().toString(), f.sourceId()); + TransformSpec spec = IcebergTableUtil.getTransformSpec(table, f.transform().toString(), f.sourceId()); spec.setFieldName(f.name()); return Pair.of(e.getKey(), spec); })) @@ -893,9 +891,8 @@ public Map> getPartitionTransformSpecs( private List getSortTransformSpec(Table table) { return table.sortOrder().fields().stream().map(s -> - IcebergTableUtil.getTransformSpec(table, s.transform().toString(), s.sourceId()) - ) - .collect(Collectors.toList()); + IcebergTableUtil.getTransformSpec(table, s.transform().toString(), s.sourceId())) + .toList(); } @Override diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index e7adf9baffd9..b292cde86500 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -91,6 +91,7 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.ResidualEvaluator; +import org.apache.iceberg.expressions.UnboundTerm; import org.apache.iceberg.hive.IcebergCatalogProperties; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.mr.Catalogs; @@ -102,6 +103,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -260,41 +262,15 @@ static PartitionStatisticsFile getPartitionStatsFile(Table table, long snapshotI * @return iceberg partition spec, always non-null */ public static PartitionSpec spec(Configuration configuration, Schema schema) { - List partitionTransformSpecList = SessionStateUtil + List partitionBy = SessionStateUtil .getResource(configuration, hive_metastoreConstants.PARTITION_TRANSFORM_SPEC) .map(o -> (List) o).orElse(null); - if (partitionTransformSpecList == null) { + if (partitionBy == null) { LOG.warn(PARTITION_TRANSFORM_SPEC_NOT_FOUND); return null; } - PartitionSpec.Builder builder = PartitionSpec.builderFor(schema); - partitionTransformSpecList.forEach(spec -> { - switch (spec.getTransformType()) { - case IDENTITY: - builder.identity(spec.getColumnName().toLowerCase()); - break; - case YEAR: - builder.year(spec.getColumnName()); - break; - case MONTH: - builder.month(spec.getColumnName()); - break; - case DAY: - builder.day(spec.getColumnName()); - break; - case HOUR: - builder.hour(spec.getColumnName()); - break; - case TRUNCATE: - builder.truncate(spec.getColumnName(), spec.getTransformParam()); - break; - case BUCKET: - builder.bucket(spec.getColumnName(), spec.getTransformParam()); - break; - } - }); - return builder.build(); + return SchemaUtils.createPartitionSpec(schema, partitionBy); } public static void updateSpec(Configuration configuration, Table table) { @@ -317,31 +293,8 @@ public static void updateSpec(Configuration configuration, Table table) { LOG.warn(PARTITION_TRANSFORM_SPEC_NOT_FOUND); return; } - partitionTransformSpecList.forEach(spec -> { - switch (spec.getTransformType()) { - case IDENTITY: - updatePartitionSpec.addField(spec.getColumnName()); - break; - case YEAR: - updatePartitionSpec.addField(Expressions.year(spec.getColumnName())); - break; - case MONTH: - updatePartitionSpec.addField(Expressions.month(spec.getColumnName())); - break; - case DAY: - updatePartitionSpec.addField(Expressions.day(spec.getColumnName())); - break; - case HOUR: - updatePartitionSpec.addField(Expressions.hour(spec.getColumnName())); - break; - case TRUNCATE: - updatePartitionSpec.addField(Expressions.truncate(spec.getColumnName(), spec.getTransformParam())); - break; - case BUCKET: - updatePartitionSpec.addField(Expressions.bucket(spec.getColumnName(), spec.getTransformParam())); - break; - } - }); + partitionTransformSpecList.forEach(spec -> + updatePartitionSpec.addField(SchemaUtils.toTerm(spec))); updatePartitionSpec.commit(); } @@ -493,24 +446,33 @@ public static PartitionData toPartitionData(StructLike sourceKey, Types.StructTy return toPartitionData(projection, targetKeyType); } - public static Expression generateExpressionFromPartitionSpec(Table table, Map partitionSpec, + public static Expression generateExpressionFromPartitionSpec(Table icebergTable, Map partitionSpec, boolean latestSpecOnly) throws SemanticException { - Map partitionFieldMap = getPartitionFields(table, latestSpecOnly).stream() - .collect(Collectors.toMap(PartitionField::name, Function.identity())); + Map partitionFieldMap = getPartitionFields(icebergTable, latestSpecOnly).stream() + .collect(Collectors.toMap( + f -> icebergTable.schema() + .findField(f.sourceId()) + .name(), + Function.identity() + )); Expression finalExp = Expressions.alwaysTrue(); for (Map.Entry entry : partitionSpec.entrySet()) { - String partColName = entry.getKey(); - if (partitionFieldMap.containsKey(partColName)) { - PartitionField partitionField = partitionFieldMap.get(partColName); - if (partitionField.transform().isIdentity()) { - final Type partKeyType = table.schema().findField(partitionField.sourceId()).type(); - final Object partKeyVal = Conversions.fromPartitionString(partKeyType, entry.getValue()); - Expression boundPredicate = Expressions.equal(partColName, partKeyVal); - finalExp = Expressions.and(finalExp, boundPredicate); - } else { - throw new SemanticException( - String.format("Partition transforms are not supported here: %s", partColName)); - } + String partColName = entry.getKey().toLowerCase(); + PartitionField partitionField = partitionFieldMap.get(partColName); + + if (partitionField != null) { + final Type sourceType = icebergTable.schema().findField(partitionField.sourceId()).type(); + final Object sourceVal = Conversions.fromPartitionString(sourceType, entry.getValue()); + // Apply the transform to the source value + @SuppressWarnings("unchecked") + Transform transform = (Transform) partitionField.transform(); + final Object partitionVal = transform.bind(sourceType).apply(sourceVal); + + TransformSpec spec = getTransformSpec(icebergTable, transform.toString(), partitionField.sourceId()); + UnboundTerm term = SchemaUtils.toTerm(spec); + + Expression boundPredicate = Expressions.equal(term, partitionVal); + finalExp = Expressions.and(finalExp, boundPredicate); } else { throw new SemanticException(String.format("No partition column by the name: %s", partColName)); } @@ -538,11 +500,11 @@ public static Partition getPartition(Configuration conf, } try { - String expectedName = Warehouse.makePartName(partitionSpec, false); - if (!expectedName.equals(partNames.getFirst())) { + Map expectedSpec = Warehouse.makeSpecFromName(partNames.getFirst()); + if (expectedSpec.size() != partitionSpec.size()) { return null; } - return new DummyPartition(table, expectedName, partitionSpec); + return new DummyPartition(table, partNames.getFirst(), partitionSpec); } catch (MetaException e) { throw new SemanticException("Unable to construct dummy partition", e); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java new file mode 100644 index 000000000000..5edbecfde32e --- /dev/null +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive; + +import java.util.List; +import org.apache.hadoop.hive.ql.parse.TransformSpec; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.UnboundTerm; + +public class SchemaUtils { + + private static final String UNSUPPORTED_TRANSFORM = "Unsupported transform: %s"; + + private SchemaUtils() { + } + + public static UnboundTerm toTerm(TransformSpec spec) { + if (spec == null) { + return null; + } + + return switch (spec.getTransformType()) { + case YEAR -> Expressions.year(spec.getColumnName()); + case MONTH -> Expressions.month(spec.getColumnName()); + case DAY -> Expressions.day(spec.getColumnName()); + case HOUR -> Expressions.hour(spec.getColumnName()); + case TRUNCATE -> Expressions.truncate(spec.getColumnName(), spec.getTransformParam()); + case BUCKET -> Expressions.bucket(spec.getColumnName(), spec.getTransformParam()); + case IDENTITY -> Expressions.ref(spec.getColumnName()); + default -> throw new UnsupportedOperationException( + UNSUPPORTED_TRANSFORM.formatted(spec.getTransformType())); + }; + } + + public static PartitionSpec createPartitionSpec(Schema schema, List partitionBy) { + if (partitionBy.isEmpty()) { + return PartitionSpec.unpartitioned(); + } + + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + + partitionBy.forEach(spec -> { + switch (spec.getTransformType()) { + case IDENTITY -> specBuilder.identity(spec.getColumnName().toLowerCase()); + case YEAR -> specBuilder.year(spec.getColumnName()); + case MONTH -> specBuilder.month(spec.getColumnName()); + case DAY -> specBuilder.day(spec.getColumnName()); + case HOUR -> specBuilder.hour(spec.getColumnName()); + case TRUNCATE -> specBuilder.truncate(spec.getColumnName(), spec.getTransformParam()); + case BUCKET -> specBuilder.bucket(spec.getColumnName(), spec.getTransformParam()); + default -> throw new UnsupportedOperationException( + UNSUPPORTED_TRANSFORM.formatted(spec.getTransformType())); + } + }); + + return specBuilder.build(); + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java index b0360112e7af..0f1a1d886fb4 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java @@ -256,4 +256,24 @@ public void testTruncateTransform() throws IOException { HiveIcebergTestUtils.validateDataWithSQL(shell, "part_test", records, "id"); } + + @Test + public void testShowPartitionsWithTransform() throws IOException { + Schema schema = new Schema( + optional(1, "a", Types.IntegerType.get()), + optional(2, "b", Types.StringType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("b", 2).build(); + List records = TestHelper.RecordsBuilder.newInstance(schema) + .add(1, "apple") + .add(2, "banana") + .build(); + testTables.createTable(shell, "test_show_parts", schema, spec, fileFormat, records); + + List rows = shell.executeStatement("SHOW PARTITIONS test_show_parts"); + Assert.assertEquals(2, rows.size()); + + rows = shell.executeStatement("SHOW PARTITIONS test_show_parts PARTITION(b='apple')"); + Assert.assertEquals(1, rows.size()); + Assert.assertEquals("b_trunc=ap", rows.get(0)[0]); + } } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q index 1246cecec7ed..e525c3808c5e 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q @@ -8,7 +8,7 @@ insert into table ice_t values( 5, "hello5" ,6, "hello6" ); desc extended ice_t PARTITION( c=6, d="hello6" ); desc formatted ice_t PARTITION( c=6, d="hello6" ); -alter table ice_t set partition spec ( c, d, b ); +alter table ice_t set partition spec ( c, d, truncate(2,b) ); insert into table ice_t values( 7, "hello7" , 8 , "hello8" ); desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ); desc formatted ice_t PARTITION( c=4 , d="hello4" ); diff --git a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out index 92f587f9b519..b3cc6ed3bcc2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out @@ -92,10 +92,10 @@ InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat Compressed: No Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] -PREHOOK: query: alter table ice_t set partition spec ( c, d, b ) +PREHOOK: query: alter table ice_t set partition spec ( c, d, truncate(2,b) ) PREHOOK: type: ALTERTABLE_SETPARTSPEC PREHOOK: Input: default@ice_t -POSTHOOK: query: alter table ice_t set partition spec ( c, d, b ) +POSTHOOK: query: alter table ice_t set partition spec ( c, d, truncate(2,b) ) POSTHOOK: type: ALTERTABLE_SETPARTSPEC POSTHOOK: Input: default@ice_t POSTHOOK: Output: default@ice_t @@ -123,13 +123,13 @@ d string # col_name data_type comment c int Transform: identity d string Transform: identity -b string Transform: identity +b string Transform: truncate[2] # Partition Transform Information # col_name transform_type c IDENTITY d IDENTITY -b IDENTITY +b TRUNCATE[2] # Detailed Partition Information Partition Value: [8, hello8, hello7] @@ -159,13 +159,13 @@ d string # col_name data_type comment c int Transform: identity d string Transform: identity -b string Transform: identity +b string Transform: truncate[2] # Partition Transform Information # col_name transform_type c IDENTITY d IDENTITY -b IDENTITY +b TRUNCATE[2] # Detailed Partition Information Partition Value: [4, hello4, null] From 80ca302cf4f9f7a6b211a5501a54289664776c65 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Thu, 22 Jan 2026 10:50:15 +0200 Subject: [PATCH 11/19] add support for partition evolution + partish stats for desc table --- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 1 - .../mr/hive/HiveIcebergStorageHandler.java | 34 ++- .../iceberg/mr/hive/IcebergTableUtil.java | 215 ++++++++++++------ .../apache/iceberg/mr/hive/SchemaUtils.java | 52 +++-- .../mr/hive/TestHiveIcebergPartitions.java | 21 +- .../positive/desc_ice_tbl_part_spec.q.out | 12 + .../formatter/TextDescTableFormatter.java | 4 +- .../hive/ql/metadata/DummyPartition.java | 6 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 12 +- 9 files changed, 244 insertions(+), 113 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index b87d56066a3a..8d8667210fb1 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -1029,7 +1029,6 @@ private static UnboundPredicate getPartitionPredicate(PartitionData part TransformSpec transformSpec = TransformSpec.fromString(field.transform().toString(), columName); UnboundTerm partitionColumn = SchemaUtils.toTerm(transformSpec); - return Expressions.equal(partitionColumn, partitionData.get(index, Object.class)); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 362c35d7d834..5b9c95a4897f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2074,8 +2074,40 @@ public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) @Override public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec, RewritePolicy policy) throws SemanticException { + validatePartSpec(table, partitionSpec, policy); - return IcebergTableUtil.getPartition(conf, table, partitionSpec); + + boolean isDescTable = SessionStateUtil.getQueryState(conf) + .map(QueryState::getHiveOperation) + .filter(op -> op == HiveOperation.DESCTABLE) + .isPresent(); + + if (!isDescTable) { + return createDummyPartitionHandle(table, partitionSpec); + } + + Partition partition = IcebergTableUtil.getPartition(conf, table, partitionSpec); + + // Populate basic statistics + if (partition != null) { + Map stats = getBasicStatistics(Partish.buildFor(table, partition)); + if (stats != null && !stats.isEmpty()) { + partition.getTPartition().setParameters(stats); + } + } + + return partition; + } + + private static DummyPartition createDummyPartitionHandle( + org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec) + throws SemanticException { + try { + String partitionName = Warehouse.makePartName(partitionSpec, false); + return new DummyPartition(table, partitionName, partitionSpec); + } catch (MetaException e) { + throw new SemanticException("Unable to construct partition name", e); + } } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index b292cde86500..026688a2230e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -105,7 +105,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.types.Conversions; -import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ByteBuffers; import org.apache.iceberg.util.Pair; @@ -295,7 +294,6 @@ public static void updateSpec(Configuration configuration, Table table) { } partitionTransformSpecList.forEach(spec -> updatePartitionSpec.addField(SchemaUtils.toTerm(spec))); - updatePartitionSpec.commit(); } @@ -446,110 +444,194 @@ public static PartitionData toPartitionData(StructLike sourceKey, Types.StructTy return toPartitionData(projection, targetKeyType); } - public static Expression generateExpressionFromPartitionSpec(Table icebergTable, Map partitionSpec, + public static Expression generateExpressionFromPartitionSpec(Table table, Map partitionSpec, boolean latestSpecOnly) throws SemanticException { - Map partitionFieldMap = getPartitionFields(icebergTable, latestSpecOnly).stream() - .collect(Collectors.toMap( - f -> icebergTable.schema() - .findField(f.sourceId()) - .name(), - Function.identity() - )); - Expression finalExp = Expressions.alwaysTrue(); + + // Group partition fields by source column name to handle partition evolution + // where the same source column may have multiple transforms across different specs + Map> partitionFieldsBySourceName = getPartitionFields(table, latestSpecOnly).stream() + .collect(Collectors.groupingBy( + partitionField -> table.schema().findColumnName(partitionField.sourceId())) + ); + + Expression predicate = Expressions.alwaysTrue(); + for (Map.Entry entry : partitionSpec.entrySet()) { - String partColName = entry.getKey().toLowerCase(); - PartitionField partitionField = partitionFieldMap.get(partColName); + String partitionColumn = entry.getKey(); + List partitionFields = partitionFieldsBySourceName.get(partitionColumn); + + if (partitionFields == null) { + throw new SemanticException(String.format("No partition column by the name: %s", partitionColumn)); + } + + // When there are multiple partition fields for the same source column (due to partition evolution), + // create an OR expression that matches any of the transforms + Expression columnPredicate = Expressions.alwaysFalse(); - if (partitionField != null) { - final Type sourceType = icebergTable.schema().findField(partitionField.sourceId()).type(); - final Object sourceVal = Conversions.fromPartitionString(sourceType, entry.getValue()); + for (PartitionField partitionField : partitionFields) { + Types.NestedField sourceField = table.schema().findField(partitionField.sourceId()); + Object sourceValue = Conversions.fromPartitionString(sourceField.type(), entry.getValue()); // Apply the transform to the source value @SuppressWarnings("unchecked") Transform transform = (Transform) partitionField.transform(); - final Object partitionVal = transform.bind(sourceType).apply(sourceVal); + Object transformedValue = transform.bind(sourceField.type()).apply(sourceValue); - TransformSpec spec = getTransformSpec(icebergTable, transform.toString(), partitionField.sourceId()); - UnboundTerm term = SchemaUtils.toTerm(spec); + TransformSpec transformSpec = TransformSpec.fromString(transform.toString().toUpperCase(), sourceField.name()); + UnboundTerm term = SchemaUtils.toTerm(transformSpec); - Expression boundPredicate = Expressions.equal(term, partitionVal); - finalExp = Expressions.and(finalExp, boundPredicate); - } else { - throw new SemanticException(String.format("No partition column by the name: %s", partColName)); + columnPredicate = Expressions.or( + columnPredicate, Expressions.equal(term, transformedValue)); } + + predicate = Expressions.and(predicate, columnPredicate); } - return finalExp; + + return predicate; } public static List getPartitionFields(Table table, boolean latestSpecOnly) { - return latestSpecOnly ? table.spec().fields() : - table.specs().values().stream() - .flatMap(spec -> spec.fields().stream() - .filter(f -> !f.transform().isVoid())) - .distinct() - .collect(Collectors.toList()); + if (latestSpecOnly) { + return table.spec().fields(); + } + return table.specs().values().stream() + .flatMap(spec -> spec.fields().stream()) + .filter(f -> !f.transform().isVoid()) + .toList(); } + /** + * Returns a partition matching the given partition spec. + * With partition evolution, multiple partitions may match; returns the one from the highest spec ID. + * @param conf Configuration + * @param table Hive table + * @param partitionSpec Partition specification with source column names and values + * @return Partition matching the spec, or null if no match found + */ public static Partition getPartition(Configuration conf, org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec) throws SemanticException { - List partNames = - getPartitionNames(conf, table, partitionSpec, false); + // Get partitions sorted by spec ID descending + List partitionNames = getPartitionNames(conf, table, partitionSpec, false, + Comparator.comparingInt((Map.Entry e) -> e.getValue()).reversed()); - if (partNames.isEmpty()) { + if (partitionNames.isEmpty()) { return null; } + // Find first partition with matching spec size (highest spec ID due to sort order) + Optional partitionName = partitionNames.stream() + .filter(p -> hasMatchingSpecSize(p, partitionSpec.size())) + .findFirst(); + + return partitionName + .map(p -> new DummyPartition(table, p, partitionSpec)) + .orElse(null); + } + + /** + * Checks if a partition name has the expected number of fields. + */ + private static boolean hasMatchingSpecSize( + String partitionName, int expectedSpecSize) { try { - Map expectedSpec = Warehouse.makeSpecFromName(partNames.getFirst()); - if (expectedSpec.size() != partitionSpec.size()) { - return null; - } - return new DummyPartition(table, partNames.getFirst(), partitionSpec); + return Warehouse.makeSpecFromName(partitionName).size() == expectedSpecSize; } catch (MetaException e) { - throw new SemanticException("Unable to construct dummy partition", e); + return false; } } /** - * Returns a list of partition names satisfying the provided partition spec. - * @param table Iceberg table - * @param partSpecMap Partition Spec used as the criteria for filtering - * @param latestSpecOnly when True, returns partitions with the current spec only, else - any specs - * @return List of partition names + * Returns partition names matching the provided partition spec. + * @param conf Configuration + * @param table Hive table + * @param partSpecMap Partition spec for filtering + * @param latestSpecOnly if true, return only partitions from latest spec; otherwise all specs + * @return List of partition names sorted by natural order */ public static List getPartitionNames(Configuration conf, org.apache.hadoop.hive.ql.metadata.Table table, Map partSpecMap, boolean latestSpecOnly) throws SemanticException { + return getPartitionNames(conf, table, partSpecMap, latestSpecOnly, Map.Entry.comparingByKey()); + } + + /** + * Returns partition names matching the provided partition spec, sorted by the given comparator. + * + * @param specIdComparator Comparator for Entry<partitionPath, specId> + */ + private static List getPartitionNames(Configuration conf, + org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec, boolean latestSpecOnly, + Comparator> specIdComparator) throws SemanticException { Table icebergTable = getTable(conf, table.getTTable()); - Expression expression = IcebergTableUtil.generateExpressionFromPartitionSpec( - icebergTable, partSpecMap, latestSpecOnly); + + Expression filterExpression = IcebergTableUtil.generateExpressionFromPartitionSpec( + icebergTable, partitionSpec, latestSpecOnly); + + int latestSpecId = icebergTable.spec().specId(); + Types.StructType partitionType = Partitioning.partitionType(icebergTable); + PartitionsTable partitionsTable = (PartitionsTable) MetadataTableUtils.createMetadataTableInstance( icebergTable, MetadataTableType.PARTITIONS); try (CloseableIterable fileScanTasks = partitionsTable.newScan().planFiles()) { return FluentIterable.from(fileScanTasks) .transformAndConcat(task -> task.asDataTask().rows()) - .transform(row -> { - StructLike data = row.get(IcebergTableUtil.PART_IDX, StructProjection.class); - PartitionSpec spec = icebergTable.specs().get(row.get(IcebergTableUtil.SPEC_IDX, Integer.class)); - return Maps.immutableEntry( - IcebergTableUtil.toPartitionData( - data, Partitioning.partitionType(icebergTable), spec.partitionType()), - spec); - }).filter(e -> { - ResidualEvaluator resEval = ResidualEvaluator.of(e.getValue(), - expression, false); - return e.getValue().isPartitioned() && - resEval.residualFor(e.getKey()).isEquivalentTo(Expressions.alwaysTrue()) && - (e.getValue().specId() == icebergTable.spec().specId() || !latestSpecOnly); - - }).transform(e -> e.getValue().partitionToPath(e.getKey())).toSortedList( - Comparator.naturalOrder()); + .transform(row -> extractPartitionDataAndSpec(row, icebergTable, partitionType)) + .filter(entry -> matchesPartition(entry, filterExpression, latestSpecOnly, latestSpecId)) + // Create (partitionPath, specId) entries for sorting + .transform(entry -> Maps.immutableEntry( + entry.getValue().partitionToPath(entry.getKey()), + entry.getValue().specId())) + .toSortedList(specIdComparator).stream() + .map(Map.Entry::getKey) + .toList(); } catch (IOException e) { - throw new SemanticException( - String.format("Error while fetching the partitions due to: %s", e)); + throw new SemanticException("Error while fetching the partitions", e); + } + } + + /** + * Checks if a partition matches the filter expression and spec requirements. + */ + private static boolean matchesPartition(Map.Entry entry, + Expression filterExpression, boolean latestSpecOnly, int latestSpecId) { + PartitionData partitionData = entry.getKey(); + PartitionSpec spec = entry.getValue(); + + // Filter unpartitioned tables + if (!spec.isPartitioned()) { + return false; + } + // Filter by spec ID if requested + if (latestSpecOnly && spec.specId() != latestSpecId) { + return false; } + // Check if partition matches filter expression + ResidualEvaluator evaluator = + ResidualEvaluator.of(spec, filterExpression, false); + + return evaluator + .residualFor(partitionData) + .isEquivalentTo(Expressions.alwaysTrue()); + } + + /** + * Extracts partition data and spec from a partitions metadata table row. + */ + private static Map.Entry extractPartitionDataAndSpec( + StructLike row, Table icebergTable, Types.StructType partitionType) { + + StructLike rawPartition = + row.get(IcebergTableUtil.PART_IDX, StructProjection.class); + + PartitionSpec spec = icebergTable.specs().get( + row.get(IcebergTableUtil.SPEC_IDX, Integer.class)); + + return Maps.immutableEntry( + IcebergTableUtil.toPartitionData( + rawPartition, partitionType, spec.partitionType()), + spec); } public static PartitionSpec getPartitionSpec(Table icebergTable, String partitionPath) @@ -581,7 +663,7 @@ public static TransformSpec getTransformSpec(Table table, String transformName, public static List readColStats(Table table, Long snapshotId, Predicate filter) { List colStats = Lists.newArrayList(); - String statsPath = IcebergTableUtil.getColStatsPath(table, snapshotId); + String statsPath = IcebergTableUtil.getColStatsPath(table, snapshotId); if (statsPath == null) { return colStats; } @@ -628,8 +710,7 @@ public static > Set getPartitionNames(Table ice Set partitions = Sets.newHashSet(); int tableSpecId = icebergTable.spec().specId(); for (T file : files) { - if (latestSpecOnly == null || latestSpecOnly && file.specId() == tableSpecId || - !latestSpecOnly && file.specId() != tableSpecId) { + if (latestSpecOnly == null || latestSpecOnly.equals(file.specId() == tableSpecId)) { String partName = icebergTable.specs().get(file.specId()).partitionToPath(file.partition()); partitions.add(partName); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java index 5edbecfde32e..8444da80510f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java @@ -39,15 +39,23 @@ public static UnboundTerm toTerm(TransformSpec spec) { } return switch (spec.getTransformType()) { - case YEAR -> Expressions.year(spec.getColumnName()); - case MONTH -> Expressions.month(spec.getColumnName()); - case DAY -> Expressions.day(spec.getColumnName()); - case HOUR -> Expressions.hour(spec.getColumnName()); - case TRUNCATE -> Expressions.truncate(spec.getColumnName(), spec.getTransformParam()); - case BUCKET -> Expressions.bucket(spec.getColumnName(), spec.getTransformParam()); - case IDENTITY -> Expressions.ref(spec.getColumnName()); - default -> throw new UnsupportedOperationException( - UNSUPPORTED_TRANSFORM.formatted(spec.getTransformType())); + case YEAR -> + Expressions.year(spec.getColumnName()); + case MONTH -> + Expressions.month(spec.getColumnName()); + case DAY -> + Expressions.day(spec.getColumnName()); + case HOUR -> + Expressions.hour(spec.getColumnName()); + case TRUNCATE -> + Expressions.truncate(spec.getColumnName(), spec.getTransformParam()); + case BUCKET -> + Expressions.bucket(spec.getColumnName(), spec.getTransformParam()); + case IDENTITY -> + Expressions.ref(spec.getColumnName()); + default -> + throw new UnsupportedOperationException( + UNSUPPORTED_TRANSFORM.formatted(spec.getTransformType())); }; } @@ -60,15 +68,23 @@ public static PartitionSpec createPartitionSpec(Schema schema, List { switch (spec.getTransformType()) { - case IDENTITY -> specBuilder.identity(spec.getColumnName().toLowerCase()); - case YEAR -> specBuilder.year(spec.getColumnName()); - case MONTH -> specBuilder.month(spec.getColumnName()); - case DAY -> specBuilder.day(spec.getColumnName()); - case HOUR -> specBuilder.hour(spec.getColumnName()); - case TRUNCATE -> specBuilder.truncate(spec.getColumnName(), spec.getTransformParam()); - case BUCKET -> specBuilder.bucket(spec.getColumnName(), spec.getTransformParam()); - default -> throw new UnsupportedOperationException( - UNSUPPORTED_TRANSFORM.formatted(spec.getTransformType())); + case IDENTITY -> + specBuilder.identity(spec.getColumnName().toLowerCase()); + case YEAR -> + specBuilder.year(spec.getColumnName()); + case MONTH -> + specBuilder.month(spec.getColumnName()); + case DAY -> + specBuilder.day(spec.getColumnName()); + case HOUR -> + specBuilder.hour(spec.getColumnName()); + case TRUNCATE -> + specBuilder.truncate(spec.getColumnName(), spec.getTransformParam()); + case BUCKET -> + specBuilder.bucket(spec.getColumnName(), spec.getTransformParam()); + default -> + throw new UnsupportedOperationException( + UNSUPPORTED_TRANSFORM.formatted(spec.getTransformType())); } }); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java index 0f1a1d886fb4..c2580c113983 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java @@ -258,22 +258,23 @@ public void testTruncateTransform() throws IOException { } @Test - public void testShowPartitionsWithTransform() throws IOException { + public void testShowPartitionsWithTransform() { Schema schema = new Schema( - optional(1, "a", Types.IntegerType.get()), - optional(2, "b", Types.StringType.get())); - PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("b", 2).build(); + optional(1, "id", Types.IntegerType.get()), + optional(2, "part_field", Types.StringType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("part_field", 2).build(); List records = TestHelper.RecordsBuilder.newInstance(schema) - .add(1, "apple") - .add(2, "banana") + .add(1L, "Part1") + .add(2L, "Part2") + .add(3L, "Art3") .build(); - testTables.createTable(shell, "test_show_parts", schema, spec, fileFormat, records); + testTables.createTable(shell, "part_test", schema, spec, fileFormat, records); - List rows = shell.executeStatement("SHOW PARTITIONS test_show_parts"); + List rows = shell.executeStatement("SHOW PARTITIONS part_test"); Assert.assertEquals(2, rows.size()); - rows = shell.executeStatement("SHOW PARTITIONS test_show_parts PARTITION(b='apple')"); + rows = shell.executeStatement("SHOW PARTITIONS part_test PARTITION(part_field='Art3')"); Assert.assertEquals(1, rows.size()); - Assert.assertEquals("b_trunc=ap", rows.get(0)[0]); + Assert.assertEquals("part_field_trunc=Ar", rows.get(0)[0]); } } diff --git a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out index b3cc6ed3bcc2..b6f0571b3d68 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out @@ -85,6 +85,10 @@ Partition Value: [6, hello6] Database: default Table: ice_t #### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1 + totalSize #Masked# # Storage Information SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe @@ -136,6 +140,10 @@ Partition Value: [8, hello8, hello7] Database: default Table: ice_t #### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1 + totalSize #Masked# # Storage Information SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe @@ -172,6 +180,10 @@ Partition Value: [4, hello4, null] Database: default Table: ice_t #### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1 + totalSize #Masked# # Storage Information SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 2847f182a8a1..add3b3c4f642 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -174,8 +174,8 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column List partitionColumns = null; if (table.isPartitioned()) { partitionColumns = table.hasNonNativePartitionSupport() ? - table.getStorageHandler().getPartitionKeys(table) : - table.getPartCols(); + table.getStorageHandler().getPartitionKeys(table) : + table.getPartCols(); } if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 1e488221e44a..0d4de06bcd8f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -84,9 +84,9 @@ public List getValues() { Table table = this.getTable(); List values = new ArrayList(); for (FieldSchema fs : - table.hasNonNativePartitionSupport() ? - table.getStorageHandler().getPartitionKeys(table) : - table.getPartCols()) { + table.hasNonNativePartitionSupport() ? + table.getStorageHandler().getPartitionKeys(table) : + table.getPartCols()) { values.add(partSpec.get(fs.getName())); } return values; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 4d85ff596b42..a0a0f7500806 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -47,7 +47,6 @@ import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.DataConnector; import org.apache.hadoop.hive.metastore.api.Database; @@ -60,7 +59,6 @@ import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.SourceTable; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.Context; @@ -89,7 +87,6 @@ import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils; import org.apache.hadoop.hive.ql.parse.type.ExprNodeTypeCheck; @@ -1357,14 +1354,7 @@ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartit if (partHandle == null) { // if partSpec doesn't exists in DB, return a delegate one // and the actual partition is created in MoveTask - try { - partHandle = tableHandle.hasNonNativePartitionSupport() ? - new DummyPartition(tableHandle, Warehouse.makePartName(partSpec, false), partSpec) : - new Partition(tableHandle, partSpec, null); - } catch (MetaException e) { - throw new SemanticException("Unable to construct name for dummy partition due to: ", e); - } - + partHandle = new Partition(tableHandle, partSpec, null); } else { partitions.add(partHandle); } From cdee454156f2b3a52725d111687380e465357245 Mon Sep 17 00:00:00 2001 From: ragupta Date: Thu, 22 Jan 2026 21:49:05 +0530 Subject: [PATCH 12/19] updated tests corresponding to transforms --- .../negative/desc_ice_tbl_partial_part_spec.q | 31 +++- .../negative/desc_ice_tbl_wrong_part_spec.q | 31 +++- .../queries/positive/desc_ice_tbl_part_spec.q | 66 +++++-- .../desc_ice_tbl_partial_part_spec.q.out | 53 +++++- .../desc_ice_tbl_wrong_part_spec.q.out | 53 +++++- .../positive/desc_ice_tbl_part_spec.q.out | 171 +++++++++++++++--- 6 files changed, 344 insertions(+), 61 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q index 577785fd6956..797a91214e1c 100644 --- a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q +++ b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q @@ -1,4 +1,27 @@ -drop table if exists ice_t; -create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; -insert into table ice_t values( 5, "hello5" ,6, "hello6"); -desc formatted ice_t PARTITION(c=6); \ No newline at end of file +DROP TABLE IF EXISTS ice_t; + +CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG; + +ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +); + +INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6"); + +-- this spec will resolve "hello5" to "he" due to truncate(2, b) which don't correspond to any partition available of size 2 +DESC FORMATTED ice_t +PARTITION (c = 6, b = "hello5"); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q index 974e61c14c1b..d58e26f20d24 100644 --- a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q +++ b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q @@ -1,4 +1,27 @@ -drop table if exists ice_t; -create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; -insert into table ice_t values( 5, "hello5" ,6, "hello6"); -desc formatted ice_t PARTITION(c=6, d="hello"); \ No newline at end of file +DROP TABLE IF EXISTS ice_t; + +CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG; + +ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +); + +INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6"); + +-- this spec will resolve "bello5" to "be" due to truncate(2, b) which don't correspond to any partition available of size 3 +DESC FORMATTED ice_t +PARTITION (c = 6, d = "hello6", b = "bello5"); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q index e525c3808c5e..451683025e73 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q @@ -1,14 +1,58 @@ -drop table if exists ice_t; -create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg; +DROP TABLE IF EXISTS ice_t; -insert into table ice_t values( 1, "hello1" ,2, "hello2" ); -insert into table ice_t values( 3, "hello3" ,4, "hello4" ); -insert into table ice_t values( 5, "hello5" ,6, "hello6" ); +CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG; -desc extended ice_t PARTITION( c=6, d="hello6" ); -desc formatted ice_t PARTITION( c=6, d="hello6" ); +INSERT INTO TABLE ice_t +VALUES (1, "hello1", 2, "hello2"); -alter table ice_t set partition spec ( c, d, truncate(2,b) ); -insert into table ice_t values( 7, "hello7" , 8 , "hello8" ); -desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ); -desc formatted ice_t PARTITION( c=4 , d="hello4" ); +INSERT INTO TABLE ice_t +VALUES (3, "hello3", 4, "hello4"); + +INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6"); + +DESC EXTENDED ice_t +PARTITION (c = 6, d = "hello6"); + +DESC FORMATTED ice_t +PARTITION (c = 6, d = "hello6"); + +ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +); + +INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8"); + +ALTER TABLE ice_t +SET PARTITION SPEC ( + bucket(16, c), + d, + truncate(2, b) +); + +INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8"); + +DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7"); + +-- this will also generate a valid result as "hello10" will resolve to "he" due to truncate(2, b) +DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello10"); + +DESC FORMATTED ice_t +PARTITION (c = 4, d = "hello4"); diff --git a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out index 0165c68d6245..7061bd176db6 100644 --- a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out +++ b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out @@ -1,23 +1,62 @@ -PREHOOK: query: drop table if exists ice_t +PREHOOK: query: DROP TABLE IF EXISTS ice_t PREHOOK: type: DROPTABLE PREHOOK: Output: database:default -POSTHOOK: query: drop table if exists ice_t +POSTHOOK: query: DROP TABLE IF EXISTS ice_t POSTHOOK: type: DROPTABLE POSTHOOK: Output: database:default -PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +PREHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@ice_t -POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +POSTHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@ice_t -PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +PREHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_t -POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t -FAILED: SemanticException [Error 10006]: Partition not found {c=6} +FAILED: SemanticException [Error 10006]: Partition not found {c=6, b=hello5} diff --git a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out index fb6e7f7acf45..206d80c2244c 100644 --- a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out +++ b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out @@ -1,23 +1,62 @@ -PREHOOK: query: drop table if exists ice_t +PREHOOK: query: DROP TABLE IF EXISTS ice_t PREHOOK: type: DROPTABLE PREHOOK: Output: database:default -POSTHOOK: query: drop table if exists ice_t +POSTHOOK: query: DROP TABLE IF EXISTS ice_t POSTHOOK: type: DROPTABLE POSTHOOK: Output: database:default -PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +PREHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@ice_t -POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +POSTHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@ice_t -PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +PREHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_t -POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6") +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t -FAILED: SemanticException [Error 10006]: Partition not found {c=6, d=hello} +FAILED: SemanticException [Error 10006]: Partition not found {c=6, d=hello6, b=bello5} diff --git a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out index b6f0571b3d68..93956d786974 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out @@ -1,45 +1,73 @@ -PREHOOK: query: drop table if exists ice_t +PREHOOK: query: DROP TABLE IF EXISTS ice_t PREHOOK: type: DROPTABLE PREHOOK: Output: database:default -POSTHOOK: query: drop table if exists ice_t +POSTHOOK: query: DROP TABLE IF EXISTS ice_t POSTHOOK: type: DROPTABLE POSTHOOK: Output: database:default -PREHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +PREHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@ice_t -POSTHOOK: query: create external table ice_t (a int, b string) partitioned by (c int, d string) write locally ordered by a desc stored by iceberg +POSTHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@ice_t -PREHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2" ) +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (1, "hello1", 2, "hello2") PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_t -POSTHOOK: query: insert into table ice_t values( 1, "hello1" ,2, "hello2" ) +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (1, "hello1", 2, "hello2") POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t -PREHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4" ) +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (3, "hello3", 4, "hello4") PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_t -POSTHOOK: query: insert into table ice_t values( 3, "hello3" ,4, "hello4" ) +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (3, "hello3", 4, "hello4") POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t -PREHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6" ) +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_t -POSTHOOK: query: insert into table ice_t values( 5, "hello5" ,6, "hello6" ) +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t -PREHOOK: query: desc extended ice_t PARTITION( c=6, d="hello6" ) +PREHOOK: query: DESC EXTENDED ice_t +PARTITION (c = 6, d = "hello6") PREHOOK: type: DESCTABLE PREHOOK: Input: default@ice_t -POSTHOOK: query: desc extended ice_t PARTITION( c=6, d="hello6" ) +POSTHOOK: query: DESC EXTENDED ice_t +PARTITION (c = 6, d = "hello6") POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@ice_t a int @@ -58,10 +86,12 @@ c IDENTITY d IDENTITY #### A masked pattern was here #### -PREHOOK: query: desc formatted ice_t PARTITION( c=6, d="hello6" ) +PREHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 6, d = "hello6") PREHOOK: type: DESCTABLE PREHOOK: Input: default@ice_t -POSTHOOK: query: desc formatted ice_t PARTITION( c=6, d="hello6" ) +POSTHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 6, d = "hello6") POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@ice_t # col_name data_type comment @@ -96,25 +126,66 @@ InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat Compressed: No Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] -PREHOOK: query: alter table ice_t set partition spec ( c, d, truncate(2,b) ) +PREHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) PREHOOK: type: ALTERTABLE_SETPARTSPEC PREHOOK: Input: default@ice_t -POSTHOOK: query: alter table ice_t set partition spec ( c, d, truncate(2,b) ) +POSTHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) POSTHOOK: type: ALTERTABLE_SETPARTSPEC POSTHOOK: Input: default@ice_t POSTHOOK: Output: default@ice_t -PREHOOK: query: insert into table ice_t values( 7, "hello7" , 8 , "hello8" ) +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8") PREHOOK: type: QUERY PREHOOK: Input: _dummy_database@_dummy_table PREHOOK: Output: default@ice_t -POSTHOOK: query: insert into table ice_t values( 7, "hello7" , 8 , "hello8" ) +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8") POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t -PREHOOK: query: desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ) +PREHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + bucket(16, c), + d, + truncate(2, b) +) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + bucket(16, c), + d, + truncate(2, b) +) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") PREHOOK: type: DESCTABLE PREHOOK: Input: default@ice_t -POSTHOOK: query: desc formatted ice_t PARTITION( c=8 , d="hello8", b="hello7" ) +POSTHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@ice_t # col_name data_type comment @@ -125,18 +196,18 @@ d string # Partition Information # col_name data_type comment -c int Transform: identity d string Transform: identity b string Transform: truncate[2] +c int Transform: bucket[16] # Partition Transform Information # col_name transform_type -c IDENTITY d IDENTITY b TRUNCATE[2] +c BUCKET[16] # Detailed Partition Information -Partition Value: [8, hello8, hello7] +Partition Value: [hello8, hello7, 8] Database: default Table: ice_t #### A masked pattern was here #### @@ -151,10 +222,12 @@ InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat Compressed: No Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] -PREHOOK: query: desc formatted ice_t PARTITION( c=4 , d="hello4" ) +PREHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello10") PREHOOK: type: DESCTABLE PREHOOK: Input: default@ice_t -POSTHOOK: query: desc formatted ice_t PARTITION( c=4 , d="hello4" ) +POSTHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello10") POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@ice_t # col_name data_type comment @@ -165,18 +238,60 @@ d string # Partition Information # col_name data_type comment -c int Transform: identity d string Transform: identity b string Transform: truncate[2] +c int Transform: bucket[16] + +# Partition Transform Information +# col_name transform_type +d IDENTITY +b TRUNCATE[2] +c BUCKET[16] + +# Detailed Partition Information +Partition Value: [hello8, hello10, 8] +Database: default +Table: ice_t +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1 + totalSize #Masked# + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] +PREHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 4, d = "hello4") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 4, d = "hello4") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +d string Transform: identity +b string Transform: truncate[2] +c int Transform: bucket[16] # Partition Transform Information # col_name transform_type -c IDENTITY d IDENTITY b TRUNCATE[2] +c BUCKET[16] # Detailed Partition Information -Partition Value: [4, hello4, null] +Partition Value: [hello4, null, 4] Database: default Table: ice_t #### A masked pattern was here #### From 6b582e0b222250ddd1ba8ad1f33dc4663c3bb049 Mon Sep 17 00:00:00 2001 From: ragupta Date: Fri, 23 Jan 2026 11:53:59 +0530 Subject: [PATCH 13/19] made partition reusable + updated throwable test --- .../TestHiveIcebergStorageHandlerNoScan.java | 24 ++++++++++++++++ .../queries/positive/desc_ice_tbl_part_spec.q | 6 ++++ .../positive/desc_ice_tbl_part_spec.q.out | 28 +++++++++++++++++++ .../table/info/desc/DescTableAnalyzer.java | 18 ++---------- .../ql/ddl/table/info/desc/DescTableDesc.java | 15 ++++++---- .../table/info/desc/DescTableOperation.java | 15 +--------- 6 files changed, 72 insertions(+), 34 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java index 343fa4429669..ca5801aea165 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java @@ -51,6 +51,7 @@ import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.BaseTable; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; @@ -1490,6 +1491,29 @@ public void testMetaHookWithUndefinedAlterOperationType() throws Exception { metaHook.commitAlterTable(hmsTable, environmentContext); } + @Test + public void testCommandsWithPartitionClause() throws IOException { + TableIdentifier target = TableIdentifier.of("default", "target"); + PartitionSpec spec = PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA) + .identity("last_name").build(); + testTables.createTable(shell, target.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, + spec, FileFormat.PARQUET, ImmutableList.of()); + PartitionData partitionData = new PartitionData(spec.partitionType()); + partitionData.set(0, "Johnson"); + org.apache.iceberg.Table icebergTable = testTables.loadTable(target); + testTables.appendIcebergTable(shell.getHiveConf(), icebergTable, FileFormat.PARQUET, partitionData, + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS); + + String[] commands = { + "DESCRIBE target PARTITION (last_name='Johnson')" + }; + + for (String command : commands) { + Assertions.assertThatCode(() -> shell.executeStatement(command)) + .doesNotThrowAnyException(); + } + } + @Test public void testAuthzURIMasked() throws TException, URISyntaxException, InterruptedException { testAuthzURI(true); diff --git a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q index 451683025e73..49948d59e5ab 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q @@ -47,6 +47,12 @@ SET PARTITION SPEC ( INSERT INTO TABLE ice_t VALUES (7, "hello7", 8, "hello8"); +EXPLAIN DESC EXTENDED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7"); + +EXPLAIN DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7"); + DESC FORMATTED ice_t PARTITION (c = 8, d = "hello8", b = "hello7"); diff --git a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out index 93956d786974..9ae5ea93e5f8 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out @@ -180,6 +180,34 @@ VALUES (7, "hello7", 8, "hello8") POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@ice_t +PREHOOK: query: EXPLAIN DESC EXTENDED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: EXPLAIN DESC EXTENDED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +Stage-1 + Fetch Operator + limit:-1 + Stage-0 + Describe Table{"table:":"default.ice_t","partition:":{"b":"hello7","c":"8","d":"hello8"},"extended:":"true"} + +PREHOOK: query: EXPLAIN DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: EXPLAIN DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +Stage-1 + Fetch Operator + limit:-1 + Stage-0 + Describe Table{"table:":"default.ice_t","partition:":{"b":"hello7","c":"8","d":"hello8"},"formatted:":"true"} + PREHOOK: query: DESC FORMATTED ice_t PARTITION (c = 8, d = "hello8", b = "hello7") PREHOOK: type: DESCTABLE diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java index 53edd29ce0be..64b6eea19171 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java @@ -77,9 +77,10 @@ public void analyzeInternal(ASTNode root) throws SemanticException { // process the second child, if exists, node to get partition spec(s) Map partitionSpec = getPartitionSpec(db, tableTypeExpr, tableName); + Partition partition = null; if (partitionSpec != null) { // validate that partition exists - PartitionUtils.getPartition(db, table, partitionSpec, true); + partition = PartitionUtils.getPartition(db, table, partitionSpec, true); } // process the third child node,if exists, to get partition spec(s) @@ -102,7 +103,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { inputs.add(new ReadEntity(table)); - DescTableDesc desc = new DescTableDesc(ctx.getResFile(), tableName, partitionSpec, columnPath, isExt, isFormatted); + DescTableDesc desc = new DescTableDesc(ctx.getResFile(), tableName, partition, columnPath, isExt, isFormatted); Task task = TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)); rootTasks.add(task); @@ -168,19 +169,6 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta } if (partitionSpec != null) { - Partition part; - try { - part = db.getPartition(table, partitionSpec); - } catch (HiveException e) { - // if get exception in finding partition it could be DESCRIBE table key - // return null, continue processing for DESCRIBE table key - return null; - } - - if (part == null) { - throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partitionSpec.toString())); - } - return partitionSpec; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java index 0354a047a901..745ca44ba3e3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -55,19 +56,19 @@ public class DescTableDesc implements DDLDesc, Serializable { private final String resFile; private final TableName tableName; - private final Map partitionSpec; + private final Partition partition; private final String columnPath; private final boolean isExtended; private final boolean isFormatted; - public DescTableDesc(Path resFile, TableName tableName, Map partitionSpec, String columnPath, - boolean isExtended, boolean isFormatted) { + public DescTableDesc(Path resFile, TableName tableName, Partition partition, + String columnPath, boolean isExtended, boolean isFormatted) { this.resFile = resFile.toString(); this.tableName = tableName; - this.partitionSpec = partitionSpec; this.columnPath = columnPath; this.isExtended = isExtended; this.isFormatted = isFormatted; + this.partition = partition; } @Explain(displayName = "result file", explainLevels = { Level.EXTENDED }) @@ -86,7 +87,7 @@ public TableName getTableName() { @Explain(displayName = "partition", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public Map getPartitionSpec() { - return partitionSpec; + return this.partition.getSpec(); } public String getColumnPath() { @@ -105,6 +106,10 @@ public boolean isFormatted() { return isFormatted; } + public Partition getPartition() { + return this.partition; + } + public static List getColumnStatisticsHeaders(boolean histogramEnabled) { ImmutableList.Builder builder = ImmutableList.builder() .add("col_name") diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index 6fef8f953ac4..6401f9a8d9b4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -26,7 +26,6 @@ import java.util.Map; import com.google.common.primitives.Longs; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.ValidTxnList; @@ -75,7 +74,7 @@ public DescTableOperation(DDLOperationContext context, DescTableDesc desc) { @Override public int execute() throws Exception { Table table = getTable(); - Partition part = getPartition(table); + Partition part = desc.getPartition(); final String dbTableName = desc.getDbTableName(); try (DataOutputStream outStream = ShowUtils.getOutputStream(new Path(desc.getResFile()), context)) { @@ -124,18 +123,6 @@ private Table getTable() throws HiveException { return table; } - private Partition getPartition(Table table) throws HiveException { - Partition part = null; - if (desc.getPartitionSpec() != null) { - part = context.getDb().getPartition(table, desc.getPartitionSpec()); - if (part == null) { - throw new HiveException(ErrorMsg.INVALID_PARTITION, - StringUtils.join(desc.getPartitionSpec().keySet(), ','), desc.getDbTableName()); - } - } - return part; - } - private Deserializer getDeserializer(Table table) throws SQLException { return table.getDeserializer(true); } From 0498459529fe565da809a44eb69ec6b4e5e87d48 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Fri, 23 Jan 2026 09:05:26 +0200 Subject: [PATCH 14/19] remove code duplication + optimization --- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 54 ++++++------------- .../mr/hive/HiveIcebergStorageHandler.java | 12 +++-- .../apache/iceberg/mr/hive/HiveTableUtil.java | 4 +- .../iceberg/mr/hive/IcebergTableUtil.java | 11 ++-- 4 files changed, 31 insertions(+), 50 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 8d8667210fb1..9574131463d3 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -20,8 +20,6 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -31,7 +29,6 @@ import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.Consumer; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.collections4.CollectionUtils; @@ -136,7 +133,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; -import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.Pair; @@ -573,41 +569,23 @@ public void rollbackAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTab @Override public void preTruncateTable(org.apache.hadoop.hive.metastore.api.Table table, EnvironmentContext context, - List partNames) - throws MetaException { + List partNames) throws MetaException { + this.tableProperties = IcebergTableProperties.getTableProperties(table, conf); this.icebergTable = Catalogs.loadTable(conf, tableProperties); - Map partitionFieldMap = icebergTable.spec().fields().stream() - .collect(Collectors.toMap(PartitionField::name, Function.identity())); - Expression finalExp = CollectionUtils.isEmpty(partNames) ? Expressions.alwaysTrue() : Expressions.alwaysFalse(); - if (partNames != null) { - for (String partName : partNames) { - Map specMap = Warehouse.makeSpecFromName(partName); - Expression subExp = Expressions.alwaysTrue(); - for (Map.Entry entry : specMap.entrySet()) { - // Since Iceberg encodes the values in UTF-8, we need to decode it. - String partColValue = URLDecoder.decode(entry.getValue(), StandardCharsets.UTF_8); - - if (partitionFieldMap.containsKey(entry.getKey())) { - PartitionField partitionField = partitionFieldMap.get(entry.getKey()); - Type resultType = partitionField.transform().getResultType(icebergTable.schema() - .findField(partitionField.sourceId()).type()); - TransformSpec.TransformType transformType = TransformSpec.fromString(partitionField.transform().toString()); - Object value = Conversions.fromPartitionString(resultType, partColValue); - Iterable iterable = () -> Collections.singletonList(value).iterator(); - if (TransformSpec.TransformType.IDENTITY.equals(transformType)) { - Expression boundPredicate = Expressions.in(partitionField.name(), iterable); - subExp = Expressions.and(subExp, boundPredicate); - } else { - throw new MetaException( - String.format("Partition transforms are not supported via truncate operation: %s", entry.getKey())); - } - } else { - throw new MetaException(String.format("No partition column/transform name by the name: %s", - entry.getKey())); - } - } - finalExp = Expressions.or(finalExp, subExp); + + Expression predicate = CollectionUtils.isEmpty(partNames) ? + Expressions.alwaysTrue() : Expressions.alwaysFalse(); + + for (String partName : partNames) { + try { + Map partSpec = Warehouse.makeSpecFromName(partName); + Expression partitionExpr = IcebergTableUtil.generateExpressionFromPartitionSpec( + icebergTable, partSpec, true); + + predicate = Expressions.or(predicate, partitionExpr); + } catch (Exception e) { + throw new MetaException("Failed to generate expression for partition: " + partName + ". " + e.getMessage()); } } @@ -616,7 +594,7 @@ public void preTruncateTable(org.apache.hadoop.hive.metastore.api.Table table, E if (branchName != null) { delete.toBranch(HiveUtils.getTableSnapshotRef(branchName)); } - delete.deleteFromRowFilter(finalExp); + delete.deleteFromRowFilter(predicate); delete.commit(); context.putToProperties("truncateSkipDataDeletion", "true"); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 5b9c95a4897f..1dd400c568ec 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2021,8 +2021,7 @@ public void validatePartSpec(org.apache.hadoop.hive.ql.metadata.Table hmsTable, * @param hmsTable A Hive table instance. * @param partitionSpec Map containing partition specification given by user. * @return true if we can perform metadata delete, otherwise false. - * @throws SemanticException Exception raised when a partition transform is being used - * or when partition column is not present in the table. + * @throws SemanticException Exception raised when partition column is not present in the table. */ @Override public boolean canUseTruncate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Map partitionSpec) @@ -2034,13 +2033,16 @@ public boolean canUseTruncate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, return false; } - Expression finalExp = IcebergTableUtil.generateExpressionFromPartitionSpec(table, partitionSpec, true); - FindFiles.Builder builder = new FindFiles.Builder(table).withRecordsMatching(finalExp); + Expression partitionExpr = IcebergTableUtil.generateExpressionFromPartitionSpec( + table, partitionSpec, true); + + FindFiles.Builder builder = new FindFiles.Builder(table).withRecordsMatching(partitionExpr); Set dataFiles = Sets.newHashSet(builder.collect()); + boolean result = true; for (DataFile dataFile : dataFiles) { PartitionData partitionData = (PartitionData) dataFile.partition(); - Expression residual = ResidualEvaluator.of(table.spec(), finalExp, false) + Expression residual = ResidualEvaluator.of(table.spec(), partitionExpr, false) .residualFor(partitionData); if (!residual.isEquivalentTo(Expressions.alwaysTrue())) { result = false; diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java index 6f572afd3b85..87eb645a8cbe 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java @@ -172,8 +172,8 @@ public static void appendFiles(URI fromURI, String format, Table icebergTbl, boo if (isOverwrite) { DeleteFiles delete = transaction.newDelete(); if (partitionSpec != null) { - Expression partitionExpr = - IcebergTableUtil.generateExpressionFromPartitionSpec(icebergTbl, partitionSpec, true); + Expression partitionExpr = IcebergTableUtil.generateExpressionFromPartitionSpec( + icebergTbl, partitionSpec, true); delete.deleteFromRowFilter(partitionExpr); } else { delete.deleteFromRowFilter(Expressions.alwaysTrue()); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 026688a2230e..72ef2a7477ed 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -466,11 +466,12 @@ public static Expression generateExpressionFromPartitionSpec(Table table, Map transform = (Transform) partitionField.transform(); @@ -512,7 +513,7 @@ public static Partition getPartition(Configuration conf, throws SemanticException { // Get partitions sorted by spec ID descending List partitionNames = getPartitionNames(conf, table, partitionSpec, false, - Comparator.comparingInt((Map.Entry e) -> e.getValue()).reversed()); + Map.Entry.comparingByValue(Comparator.reverseOrder())); if (partitionNames.isEmpty()) { return null; @@ -564,7 +565,7 @@ private static List getPartitionNames(Configuration conf, Comparator> specIdComparator) throws SemanticException { Table icebergTable = getTable(conf, table.getTTable()); - Expression filterExpression = IcebergTableUtil.generateExpressionFromPartitionSpec( + Expression partitionExpr = IcebergTableUtil.generateExpressionFromPartitionSpec( icebergTable, partitionSpec, latestSpecOnly); int latestSpecId = icebergTable.spec().specId(); @@ -577,7 +578,7 @@ private static List getPartitionNames(Configuration conf, return FluentIterable.from(fileScanTasks) .transformAndConcat(task -> task.asDataTask().rows()) .transform(row -> extractPartitionDataAndSpec(row, icebergTable, partitionType)) - .filter(entry -> matchesPartition(entry, filterExpression, latestSpecOnly, latestSpecId)) + .filter(entry -> matchesPartition(entry, partitionExpr, latestSpecOnly, latestSpecId)) // Create (partitionPath, specId) entries for sorting .transform(entry -> Maps.immutableEntry( entry.getValue().partitionToPath(entry.getKey()), From aaf273c271e59a232ea512c51ccfa47bfcfdd313 Mon Sep 17 00:00:00 2001 From: ragupta Date: Fri, 23 Jan 2026 16:11:40 +0530 Subject: [PATCH 15/19] remove redundant calculation of values + added todo for leftover --- .../formatter/TextDescTableFormatter.java | 1 + .../hive/ql/metadata/DummyPartition.java | 36 +++++++++++-------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index add3b3c4f642..584985310bb2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -172,6 +172,7 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column String partitionData = ""; if (columnPath == null) { List partitionColumns = null; + // @TODO: HIVE-29413 : implement generic implemntation of getPartCols() if (table.isPartitioned()) { partitionColumns = table.hasNonNativePartitionSupport() ? table.getStorageHandler().getPartitionKeys(table) : diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 0d4de06bcd8f..4ff7fd4b500a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -38,25 +38,27 @@ public class DummyPartition extends Partition { private String name; private LinkedHashMap partSpec; + private List values; + public DummyPartition() { } - public DummyPartition(Table tbl, String name) { - setTable(tbl); + public DummyPartition(Table table, String name) { + setTable(table); this.name = name; } - public DummyPartition(Table tbl) { - this(tbl, null, Maps.newHashMap()); + public DummyPartition(Table table) { + this(table, null, Maps.newHashMap()); } - public DummyPartition(Table tbl, String name, Map partSpec) { - this(tbl, name); + public DummyPartition(Table table, String name, Map partSpec) { + this(table, name); org.apache.hadoop.hive.metastore.api.Partition tPart = new org.apache.hadoop.hive.metastore.api.Partition(); - tPart.setSd(tbl.getSd().deepCopy()); + tPart.setSd(table.getSd().deepCopy()); tPart.setParameters(Maps.newHashMap()); - tPart.setDbName(tbl.getDbName()); + tPart.setDbName(table.getDbName()); this.partSpec = Maps.newLinkedHashMap(partSpec); setTPartition(tPart); @@ -81,14 +83,20 @@ public LinkedHashMap getSpec() { @Override public List getValues() { + if (values != null) { + return values; + } + Table table = this.getTable(); - List values = new ArrayList(); - for (FieldSchema fs : - table.hasNonNativePartitionSupport() ? - table.getStorageHandler().getPartitionKeys(table) : - table.getPartCols()) { - values.add(partSpec.get(fs.getName())); + values = new ArrayList<>(); + + for (FieldSchema fs : table.hasNonNativePartitionSupport() + ? table.getStorageHandler().getPartitionKeys(table) + : table.getPartCols()) { + String val = partSpec.get(fs.getName()); + values.add(val); } + return values; } From 3e94b00ae1dad756bfa04af40bf300b7fa71745c Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Fri, 23 Jan 2026 18:10:23 +0200 Subject: [PATCH 16/19] truncte fix --- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 9574131463d3..19c8b68b8272 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -574,8 +574,25 @@ public void preTruncateTable(org.apache.hadoop.hive.metastore.api.Table table, E this.tableProperties = IcebergTableProperties.getTableProperties(table, conf); this.icebergTable = Catalogs.loadTable(conf, tableProperties); - Expression predicate = CollectionUtils.isEmpty(partNames) ? - Expressions.alwaysTrue() : Expressions.alwaysFalse(); + Expression predicate = buildDeletePredicate(partNames); + + DeleteFiles delete = icebergTable.newDelete(); + String branchName = context.getProperties().get(Catalogs.SNAPSHOT_REF); + if (branchName != null) { + delete.toBranch(HiveUtils.getTableSnapshotRef(branchName)); + } + + delete.deleteFromRowFilter(predicate); + delete.commit(); + context.putToProperties("truncateSkipDataDeletion", "true"); + } + + private Expression buildDeletePredicate(List partNames) throws MetaException { + if (CollectionUtils.isEmpty(partNames)) { + return Expressions.alwaysTrue(); + } + + Expression predicate = Expressions.alwaysFalse(); for (String partName : partNames) { try { @@ -585,18 +602,12 @@ public void preTruncateTable(org.apache.hadoop.hive.metastore.api.Table table, E predicate = Expressions.or(predicate, partitionExpr); } catch (Exception e) { - throw new MetaException("Failed to generate expression for partition: " + partName + ". " + e.getMessage()); + throw new MetaException( + "Failed to generate expression for partition: " + partName + ". " + e.getMessage()); } } - DeleteFiles delete = icebergTable.newDelete(); - String branchName = context.getProperties().get(Catalogs.SNAPSHOT_REF); - if (branchName != null) { - delete.toBranch(HiveUtils.getTableSnapshotRef(branchName)); - } - delete.deleteFromRowFilter(predicate); - delete.commit(); - context.putToProperties("truncateSkipDataDeletion", "true"); + return predicate; } @Override public boolean createHMSTableInHook() { From 3e4698636c56a42b1d431aae15b9f92b43baf293 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Fri, 23 Jan 2026 20:09:19 +0200 Subject: [PATCH 17/19] desctableanalyzer refactor --- .../table/info/desc/DescTableAnalyzer.java | 52 +++++++++---------- .../ql/ddl/table/info/desc/DescTableDesc.java | 16 +++--- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java index 64b6eea19171..c5315f83bc04 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.ddl.DDLWork; -import org.apache.hadoop.hive.ql.ddl.table.partition.PartitionUtils; import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType; import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.exec.Task; @@ -32,7 +31,6 @@ import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -75,16 +73,11 @@ public void analyzeInternal(ASTNode root) throws SemanticException { } Table table = getTable(tableName); - // process the second child, if exists, node to get partition spec(s) - Map partitionSpec = getPartitionSpec(db, tableTypeExpr, tableName); - Partition partition = null; - if (partitionSpec != null) { - // validate that partition exists - partition = PartitionUtils.getPartition(db, table, partitionSpec, true); - } + // process the second child, if exists, node to get partition + Partition partition = getPartition(db, tableTypeExpr, table); // process the third child node,if exists, to get partition spec(s) - String columnPath = getColumnPath(tableTypeExpr, tableName, partitionSpec); + String columnPath = getColumnPath(tableTypeExpr, tableName, partition); boolean showColStats = false; boolean isFormatted = false; @@ -118,7 +111,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { * Example: lintString.$elem$.myint. * Return table name for column name if no column has been specified. */ - private String getColumnPath(ASTNode node, TableName tableName, Map partitionSpec) { + private String getColumnPath(ASTNode node, TableName tableName, Partition partition) { // if this ast has only one child, then no column name specified. if (node.getChildCount() == 1) { return null; @@ -126,7 +119,8 @@ private String getColumnPath(ASTNode node, TableName tableName, Map 1) { - ASTNode columnNode = (partitionSpec == null) ? (ASTNode) node.getChild(1) : (ASTNode) node.getChild(2); + ASTNode columnNode = (partition == null) ? + (ASTNode) node.getChild(1) : (ASTNode) node.getChild(2); if (columnNode != null) { return String.join(".", tableName.getNotEmptyDbTable(), DDLUtils.getFQName(columnNode)); } @@ -135,7 +129,8 @@ private String getColumnPath(ASTNode node, TableName tableName, Map getPartitionSpec(Hive db, ASTNode node, TableName tableName) throws SemanticException { + private Partition getPartition(Hive db, ASTNode node, Table table) + throws SemanticException { // if this node has only one child, then no partition spec specified. if (node.getChildCount() == 1) { return null; @@ -150,28 +145,33 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta if (node.getChild(1).getType() == HiveParser.TOK_PARTSPEC) { ASTNode partNode = (ASTNode) node.getChild(1); - Table table; - try { - table = db.getTable(tableName.getNotEmptyDbTable()); - } catch (InvalidTableException e) { - throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName.getNotEmptyDbTable()), e); - } catch (HiveException e) { - throw new SemanticException(e.getMessage(), e); - } - Map partitionSpec; try { partitionSpec = getValidatedPartSpec(table, partNode, db.getConf(), false); } catch (SemanticException e) { - // get exception in resolving partition it could be DESCRIBE table key - // return null, continue processing for DESCRIBE table key + // continue processing for DESCRIBE table key + return null; + } + + if (partitionSpec == null || partitionSpec.isEmpty()) { return null; } - if (partitionSpec != null) { - return partitionSpec; + Partition partition; + try { + partition = db.getPartition(table, partitionSpec); + } catch (HiveException e) { + // continue processing for DESCRIBE table key + return null; } + + if (partition == null) { + throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partitionSpec.toString())); + } + + return partition; } + // If child(1) is not TOK_PARTSPEC, it's a column name, return null return null; } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java index 745ca44ba3e3..f0d979e80474 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java @@ -61,14 +61,14 @@ public class DescTableDesc implements DDLDesc, Serializable { private final boolean isExtended; private final boolean isFormatted; - public DescTableDesc(Path resFile, TableName tableName, Partition partition, - String columnPath, boolean isExtended, boolean isFormatted) { + public DescTableDesc(Path resFile, TableName tableName, Partition partition, String columnPath, + boolean isExtended, boolean isFormatted) { this.resFile = resFile.toString(); this.tableName = tableName; + this.partition = partition; this.columnPath = columnPath; this.isExtended = isExtended; this.isFormatted = isFormatted; - this.partition = partition; } @Explain(displayName = "result file", explainLevels = { Level.EXTENDED }) @@ -85,9 +85,13 @@ public TableName getTableName() { return tableName; } + public Partition getPartition() { + return partition; + } + @Explain(displayName = "partition", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public Map getPartitionSpec() { - return this.partition.getSpec(); + return partition != null ? partition.getSpec() : null; } public String getColumnPath() { @@ -106,10 +110,6 @@ public boolean isFormatted() { return isFormatted; } - public Partition getPartition() { - return this.partition; - } - public static List getColumnStatisticsHeaders(boolean histogramEnabled) { ImmutableList.Builder builder = ImmutableList.builder() .add("col_name") From 19f421eecc9f423f990e64f179fcdfaf648978b8 Mon Sep 17 00:00:00 2001 From: Denys Kuzmenko Date: Sat, 24 Jan 2026 14:22:32 +0200 Subject: [PATCH 18/19] truncate optimization (cherry picked from commit 67440d331ebcec0ca189710d1bba787b9096a77b) --- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 4 +++- .../apache/iceberg/mr/hive/IcebergTableUtil.java | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 19c8b68b8272..986b9cd11e6e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -592,13 +592,15 @@ private Expression buildDeletePredicate(List partNames) throws MetaExcep return Expressions.alwaysTrue(); } + Map> partitionFieldsBySourceColumn = + IcebergTableUtil.partitionFieldsBySourceColumn(icebergTable, true); Expression predicate = Expressions.alwaysFalse(); for (String partName : partNames) { try { Map partSpec = Warehouse.makeSpecFromName(partName); Expression partitionExpr = IcebergTableUtil.generateExpressionFromPartitionSpec( - icebergTable, partSpec, true); + icebergTable, partSpec, partitionFieldsBySourceColumn); predicate = Expressions.or(predicate, partitionExpr); } catch (Exception e) { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 72ef2a7477ed..90e92c755b9a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -447,18 +447,27 @@ public static PartitionData toPartitionData(StructLike sourceKey, Types.StructTy public static Expression generateExpressionFromPartitionSpec(Table table, Map partitionSpec, boolean latestSpecOnly) throws SemanticException { + return generateExpressionFromPartitionSpec( + table, partitionSpec, partitionFieldsBySourceColumn(table, latestSpecOnly)); + } + + public static Map> partitionFieldsBySourceColumn(Table table, boolean latestSpecOnly) { // Group partition fields by source column name to handle partition evolution // where the same source column may have multiple transforms across different specs - Map> partitionFieldsBySourceName = getPartitionFields(table, latestSpecOnly).stream() + return getPartitionFields(table, latestSpecOnly).stream() .collect(Collectors.groupingBy( partitionField -> table.schema().findColumnName(partitionField.sourceId())) ); + } + + public static Expression generateExpressionFromPartitionSpec(Table table, Map partitionSpec, + Map> partitionFieldsBySourceColumn) throws SemanticException { Expression predicate = Expressions.alwaysTrue(); for (Map.Entry entry : partitionSpec.entrySet()) { String partitionColumn = entry.getKey(); - List partitionFields = partitionFieldsBySourceName.get(partitionColumn); + List partitionFields = partitionFieldsBySourceColumn.get(partitionColumn); if (partitionFields == null) { throw new SemanticException(String.format("No partition column by the name: %s", partitionColumn)); @@ -466,7 +475,8 @@ public static Expression generateExpressionFromPartitionSpec(Table table, Map Date: Sat, 24 Jan 2026 20:08:06 +0200 Subject: [PATCH 19/19] restore identitypartitionexpt for truncate, etc --- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 19 ++- .../mr/hive/HiveIcebergStorageHandler.java | 2 +- .../apache/iceberg/mr/hive/HiveTableUtil.java | 2 +- .../iceberg/mr/hive/IcebergTableUtil.java | 115 +++++++++++++----- .../formatter/TextDescTableFormatter.java | 2 +- .../hive/ql/metadata/DummyPartition.java | 1 + 6 files changed, 95 insertions(+), 46 deletions(-) diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 986b9cd11e6e..865116414459 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.collections4.CollectionUtils; @@ -574,7 +575,7 @@ public void preTruncateTable(org.apache.hadoop.hive.metastore.api.Table table, E this.tableProperties = IcebergTableProperties.getTableProperties(table, conf); this.icebergTable = Catalogs.loadTable(conf, tableProperties); - Expression predicate = buildDeletePredicate(partNames); + Expression predicate = generateExprFromPartitionNames(partNames); DeleteFiles delete = icebergTable.newDelete(); String branchName = context.getProperties().get(Catalogs.SNAPSHOT_REF); @@ -587,20 +588,20 @@ public void preTruncateTable(org.apache.hadoop.hive.metastore.api.Table table, E context.putToProperties("truncateSkipDataDeletion", "true"); } - private Expression buildDeletePredicate(List partNames) throws MetaException { + private Expression generateExprFromPartitionNames(List partNames) throws MetaException { if (CollectionUtils.isEmpty(partNames)) { return Expressions.alwaysTrue(); } - Map> partitionFieldsBySourceColumn = - IcebergTableUtil.partitionFieldsBySourceColumn(icebergTable, true); + Map partitionFields = icebergTable.spec().fields().stream() + .collect(Collectors.toMap(PartitionField::name, Function.identity())); Expression predicate = Expressions.alwaysFalse(); for (String partName : partNames) { try { - Map partSpec = Warehouse.makeSpecFromName(partName); - Expression partitionExpr = IcebergTableUtil.generateExpressionFromPartitionSpec( - icebergTable, partSpec, partitionFieldsBySourceColumn); + Map partitionSpec = Warehouse.makeSpecFromName(partName); + Expression partitionExpr = IcebergTableUtil.generateExprForIdentityPartition( + icebergTable, partitionSpec, partitionFields); predicate = Expressions.or(predicate, partitionExpr); } catch (Exception e) { @@ -612,10 +613,6 @@ private Expression buildDeletePredicate(List partNames) throws MetaExcep return predicate; } - @Override public boolean createHMSTableInHook() { - return createHMSTableInHook; - } - private void alterTableProperties(org.apache.hadoop.hive.metastore.api.Table hmsTable, Map contextProperties) { Map hmsTableParameters = hmsTable.getParameters(); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 1dd400c568ec..7554de2c588a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -2033,7 +2033,7 @@ public boolean canUseTruncate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, return false; } - Expression partitionExpr = IcebergTableUtil.generateExpressionFromPartitionSpec( + Expression partitionExpr = IcebergTableUtil.generateExprForIdentityPartition( table, partitionSpec, true); FindFiles.Builder builder = new FindFiles.Builder(table).withRecordsMatching(partitionExpr); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java index 87eb645a8cbe..fbdc36be3a19 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java @@ -172,7 +172,7 @@ public static void appendFiles(URI fromURI, String format, Table icebergTbl, boo if (isOverwrite) { DeleteFiles delete = transaction.newDelete(); if (partitionSpec != null) { - Expression partitionExpr = IcebergTableUtil.generateExpressionFromPartitionSpec( + Expression partitionExpr = IcebergTableUtil.generateExprForIdentityPartition( icebergTbl, partitionSpec, true); delete.deleteFromRowFilter(partitionExpr); } else { diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 90e92c755b9a..b85df34405d0 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -440,61 +440,112 @@ public static PartitionData toPartitionData(StructLike key, Types.StructType key public static PartitionData toPartitionData(StructLike sourceKey, Types.StructType sourceKeyType, Types.StructType targetKeyType) { - StructProjection projection = StructProjection.create(sourceKeyType, targetKeyType).wrap(sourceKey); + StructProjection projection = StructProjection.create(sourceKeyType, targetKeyType) + .wrap(sourceKey); return toPartitionData(projection, targetKeyType); } - public static Expression generateExpressionFromPartitionSpec(Table table, Map partitionSpec, + public static Expression generateExprForIdentityPartition(Table table, Map partitionSpec, boolean latestSpecOnly) throws SemanticException { - return generateExpressionFromPartitionSpec( - table, partitionSpec, partitionFieldsBySourceColumn(table, latestSpecOnly)); + Map partitionFields = getPartitionFields(table, latestSpecOnly).stream() + .collect(Collectors.toMap(PartitionField::name, Function.identity())); + + return generateExprForIdentityPartition(table, partitionSpec, partitionFields); + } + + public static Expression generateExprForIdentityPartition(Table table, Map partitionSpec, + Map partitionFields) throws SemanticException { + + return buildPartitionExpression( + partitionSpec, + (column, value) -> + buildIdentityPartitionPredicate(table, value, partitionFields.get(column)), + partitionFields::containsKey + ); } - public static Map> partitionFieldsBySourceColumn(Table table, boolean latestSpecOnly) { + public static Expression generateExprFromPartitionSpec(Table table, Map partitionSpec, + boolean latestSpecOnly) throws SemanticException { + // Group partition fields by source column name to handle partition evolution // where the same source column may have multiple transforms across different specs - return getPartitionFields(table, latestSpecOnly).stream() - .collect(Collectors.groupingBy( - partitionField -> table.schema().findColumnName(partitionField.sourceId())) - ); + Map> partitionFieldsBySourceColumn = + getPartitionFields(table, latestSpecOnly).stream() + .collect(Collectors.groupingBy( + pf -> table.schema().findColumnName(pf.sourceId())) + ); + + return buildPartitionExpression( + partitionSpec, + (column, value) -> + buildTransformPartitionPredicate(table, value, partitionFieldsBySourceColumn.get(column)), + partitionFieldsBySourceColumn::containsKey + ); } - public static Expression generateExpressionFromPartitionSpec(Table table, Map partitionSpec, - Map> partitionFieldsBySourceColumn) throws SemanticException { + @FunctionalInterface + private interface PartitionPredicateBuilder { + Expression build(String partitionColumn, String partitionValue) throws SemanticException; + } + + private static Expression buildPartitionExpression( + Map partitionSpec, + PartitionPredicateBuilder predicateBuilder, + Predicate fieldValidator) throws SemanticException { Expression predicate = Expressions.alwaysTrue(); for (Map.Entry entry : partitionSpec.entrySet()) { String partitionColumn = entry.getKey(); - List partitionFields = partitionFieldsBySourceColumn.get(partitionColumn); - if (partitionFields == null) { - throw new SemanticException(String.format("No partition column by the name: %s", partitionColumn)); + // Validate field exists + if (!fieldValidator.test(partitionColumn)) { + throw new SemanticException( + "No partition column by the name: %s".formatted(partitionColumn)); } + Expression columnPredicate = predicateBuilder.build(partitionColumn, entry.getValue()); + predicate = Expressions.and(predicate, columnPredicate); + } - // When there are multiple partition fields for the same source column (due to partition evolution), - // create an OR expression that matches any of the transforms - Types.NestedField sourceField = table.schema().findField( - partitionFields.getFirst().sourceId()); - Object sourceValue = Conversions.fromPartitionString(sourceField.type(), entry.getValue()); + return predicate; + } - Expression columnPredicate = Expressions.alwaysFalse(); + private static Expression buildIdentityPartitionPredicate(Table table, String partitionValue, + PartitionField partitionField) throws SemanticException { - for (PartitionField partitionField : partitionFields) { - // Apply the transform to the source value - @SuppressWarnings("unchecked") - Transform transform = (Transform) partitionField.transform(); - Object transformedValue = transform.bind(sourceField.type()).apply(sourceValue); + if (!partitionField.transform().isIdentity()) { + throw new SemanticException( + "Partition transforms are not supported here: %s".formatted(partitionField.name())); + } + Types.NestedField sourceField = table.schema().findField(partitionField.sourceId()); + Object columnValue = Conversions.fromPartitionString(sourceField.type(), partitionValue); - TransformSpec transformSpec = TransformSpec.fromString(transform.toString().toUpperCase(), sourceField.name()); - UnboundTerm term = SchemaUtils.toTerm(transformSpec); + return Expressions.equal(partitionField.name(), columnValue); + } - columnPredicate = Expressions.or( - columnPredicate, Expressions.equal(term, transformedValue)); - } + private static Expression buildTransformPartitionPredicate(Table table, String partitionValue, + List partitionFields) { - predicate = Expressions.and(predicate, columnPredicate); + // Get source field type from first partition field (all share same source) + Types.NestedField sourceField = table.schema().findField( + partitionFields.getFirst().sourceId()); + Object columnValue = Conversions.fromPartitionString(sourceField.type(), partitionValue); + + Expression predicate = Expressions.alwaysFalse(); + + // Create OR expression for each transform on this source column + for (PartitionField partitionField : partitionFields) { + // Apply the transform to the source value + @SuppressWarnings("unchecked") + Transform transform = (Transform) partitionField.transform(); + Object transformedValue = transform.bind(sourceField.type()).apply(columnValue); + + TransformSpec transformSpec = TransformSpec.fromString(transform.toString().toUpperCase(), sourceField.name()); + UnboundTerm term = SchemaUtils.toTerm(transformSpec); + + predicate = Expressions.or( + predicate, Expressions.equal(term, transformedValue)); } return predicate; @@ -575,7 +626,7 @@ private static List getPartitionNames(Configuration conf, Comparator> specIdComparator) throws SemanticException { Table icebergTable = getTable(conf, table.getTTable()); - Expression partitionExpr = IcebergTableUtil.generateExpressionFromPartitionSpec( + Expression partitionExpr = IcebergTableUtil.generateExprFromPartitionSpec( icebergTable, partitionSpec, latestSpecOnly); int latestSpecId = icebergTable.spec().specId(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 584985310bb2..b1dd9738572a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -172,7 +172,7 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column String partitionData = ""; if (columnPath == null) { List partitionColumns = null; - // @TODO: HIVE-29413 : implement generic implemntation of getPartCols() + // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation if (table.isPartitioned()) { partitionColumns = table.hasNonNativePartitionSupport() ? table.getStorageHandler().getPartitionKeys(table) : diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 4ff7fd4b500a..c188eb09fdcf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -90,6 +90,7 @@ public List getValues() { Table table = this.getTable(); values = new ArrayList<>(); + // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation for (FieldSchema fs : table.hasNonNativePartitionSupport() ? table.getStorageHandler().getPartitionKeys(table) : table.getPartCols()) {