From 2580044be65b9457f28d163ec724ce18410de4e0 Mon Sep 17 00:00:00 2001 From: ramitg254 <97451167+ramitg254@users.noreply.github.com> Date: Tue, 16 Jun 2026 18:13:38 +0530 Subject: [PATCH] HIVE-29669: EXPLAIN DDL fails on ACID tables with "Cannot find valid write ID list" when HIVE_AUTHORIZATION_ENABLED is enabled Change-Id: I822b08acd8e3cc0c7264cc4422bddee28e57cccd --- .../hadoop/hive/ql/exec/DDLPlanUtils.java | 21 +++-- .../hadoop/hive/ql/exec/ExplainTask.java | 14 ++-- .../hadoop/hive/ql/TestTxnCommands.java | 2 +- .../hadoop/hive/ql/TestTxnCommands2.java | 2 +- ...2WithAbortCleanupUsingCompactionCycle.java | 2 +- ...mands2WithSplitUpdateAndVectorization.java | 2 +- .../hive/ql/TestTxnCommandsForMmTable.java | 2 +- ...mmandsWithSplitUpdateAndVectorization.java | 2 +- .../hive/ql/TxnCommandsBaseForTests.java | 2 +- .../hive/ql/exec/TestExplainDdlAcidTable.java | 76 +++++++++++++++++++ 10 files changed, 106 insertions(+), 19 deletions(-) create mode 100644 ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainDdlAcidTable.java diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java index a5bc66733f46..0a59ef470758 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java @@ -26,6 +26,7 @@ import com.google.common.collect.Sets; import java.util.Comparator; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; @@ -93,6 +94,16 @@ import static org.apache.hadoop.hive.serde.serdeConstants.UNION_TYPE_NAME; public class DDLPlanUtils { + private final HiveConf conf; + + public DDLPlanUtils() { + this(null); + } + + public DDLPlanUtils(HiveConf conf) { + this.conf = conf; + } + private static final String EXTERNAL = "external"; private static final String TEMPORARY = "temporary"; private static final String LIST_COLUMNS = "columns"; @@ -239,7 +250,7 @@ public class DDLPlanUtils { + TABLE_NAME + "> PARTITION <" + PARTITION_NAME + "> FOR COLUMN <" + COLUMN_NAME + "> BUT IT IS NOT SUPPORTED YET. THE BASE64 VALUE FOR THE HISTOGRAM IS <" + BASE_64_VALUE + "> "; - + /** * Returns the create database query for a give database name. * @@ -522,7 +533,7 @@ public List getAlterTableStmtTableStatsColsAll(Table tbl) throws HiveException { List alterTblStmt = new ArrayList<>(); List accessedColumns = getTableColumnNames(tbl); - List tableColumnStatistics = Hive.get().getTableColumnStatistics( + List tableColumnStatistics = Hive.get(conf).getTableColumnStatistics( tbl, accessedColumns, true); ColumnStatisticsObj[] columnStatisticsObj = tableColumnStatistics.toArray(new ColumnStatisticsObj[0]); @@ -663,9 +674,9 @@ public List getDDLPlanForPartitionWithStats(Table table, List columnNames = getTableColumnNames(table); tableToPartitionList.get(tableName).forEach(p -> partNames.add(p.getName())); Map> partitionColStats = - Hive.get().getPartitionColumnStatistics(databaseName, - tableName, partNames, columnNames, - true); + Hive.get(conf).getPartitionColumnStatistics(databaseName, + tableName, partNames, columnNames, + true); Map partitionToActualName = new HashMap<>(); tableToPartitionList.get(tableName).forEach(p -> partitionToActualName.put(p.getName(), getPartitionActualName(p))); partitionColStats.keySet().stream().sorted().forEach(partitionName -> diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index 703b92fa6270..e615e6d2798e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -488,13 +488,13 @@ public void addExplain(String sql , List explainStmt, DDLPlanUtils ddlPl } public void getDDLPlan(PrintStream out) throws Exception { - DDLPlanUtils ddlPlanUtils = new DDLPlanUtils(); - Set createDatabase = new TreeSet(); - List tableCreateStmt = new LinkedList(); - List tableBasicDef = new LinkedList(); - List createViewList = new LinkedList(); - List alterTableStmt = new LinkedList(); - List explainStmt = new LinkedList(); + DDLPlanUtils ddlPlanUtils = new DDLPlanUtils(conf); + Set createDatabase = new TreeSet<>(); + List tableCreateStmt = new LinkedList<>(); + List tableBasicDef = new LinkedList<>(); + List createViewList = new LinkedList<>(); + List alterTableStmt = new LinkedList<>(); + List explainStmt = new LinkedList<>(); Map tableMap = new HashMap<>(); Map> tablePartitionsMap = new HashMap<>(); for (ReadEntity ent : work.getInputs()) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java index 01dd3d51f5fa..f547b9366632 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java @@ -118,7 +118,7 @@ protected String getTestDataDir() { } @Override - void initHiveConf() { + protected void initHiveConf() { super.initHiveConf(); //TestTxnCommandsWithSplitUpdateAndVectorization has the vectorized version //of these tests. diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index a74d3170b6a2..cd135d93130d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -155,7 +155,7 @@ String getPartitionColumns() { public ExpectedException expectedException = ExpectedException.none(); @Override - void initHiveConf() { + protected void initHiveConf() { super.initHiveConf(); //TestTxnCommands2WithSplitUpdateAndVectorization has the vectorized version //of these tests. diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithAbortCleanupUsingCompactionCycle.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithAbortCleanupUsingCompactionCycle.java index 628c0f979aaa..f653b9ff2635 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithAbortCleanupUsingCompactionCycle.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithAbortCleanupUsingCompactionCycle.java @@ -30,7 +30,7 @@ public TestTxnCommands2WithAbortCleanupUsingCompactionCycle() { } @Override - void initHiveConf() { + protected void initHiveConf() { super.initHiveConf(); MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER, false); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdateAndVectorization.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdateAndVectorization.java index 55cf77b99d31..ce1c29e0a82b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdateAndVectorization.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdateAndVectorization.java @@ -34,7 +34,7 @@ public TestTxnCommands2WithSplitUpdateAndVectorization() { } @Override - void initHiveConf() { + protected void initHiveConf() { super.initHiveConf(); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java index 5b243d2022b1..ad821ba74f68 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java @@ -74,7 +74,7 @@ public String toString() { } @Override - void initHiveConf() { + protected void initHiveConf() { super.initHiveConf(); HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_ACID_TRUNCATE_USE_BASE, false); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsWithSplitUpdateAndVectorization.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsWithSplitUpdateAndVectorization.java index a0132300257a..e05042794ef4 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsWithSplitUpdateAndVectorization.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsWithSplitUpdateAndVectorization.java @@ -29,7 +29,7 @@ public TestTxnCommandsWithSplitUpdateAndVectorization() { } @Override - void initHiveConf() { + protected void initHiveConf() { super.initHiveConf(); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java index ba093d0ac963..097f5751c646 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java @@ -116,7 +116,7 @@ public void setUp() throws Exception { HiveMetaStoreClientWithLocalCache.init(hiveConf); } } - void initHiveConf() { + protected void initHiveConf() { hiveConf = new HiveConfForTest(this.getClass()); // Multiple tests requires more than one buckets per write. Use a very small value for grouping size to create // multiple mapper instances with FileSinkOperators. The number of buckets are depends on the size of the data diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainDdlAcidTable.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainDdlAcidTable.java new file mode 100644 index 000000000000..bc11d5adb586 --- /dev/null +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExplainDdlAcidTable.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import java.io.File; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.TxnCommandsBaseForTests; +import org.junit.Test; + +public class TestExplainDdlAcidTable extends TxnCommandsBaseForTests { + + private static final String TEST_DATA_DIR = new File(System.getProperty("java.io.tmpdir") + + File.separator + TestExplainDdlAcidTable.class.getCanonicalName() + + "-" + System.currentTimeMillis() + ).getPath().replaceAll("\\\\", "/"); + + private static final String EXPLAIN_DDL = "EXPLAIN DDL SELECT * FROM " + Table.ACIDTBL; + + @Override + protected void initHiveConf() { + super.initHiveConf(); + HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.HIVE_IN_TEST, false); + } + + @Override + protected String getTestDataDir() { + return TEST_DATA_DIR; + } + + @Test + public void testExplainDdlAcidTableUnauthorized() throws Exception { + runExplainDdl(hiveConf); + } + + /** + * {@link DDLPlanUtils} must use the query conf. + * HIVE-29330 repoints thread-local {@link org.apache.hadoop.hive.ql.metadata.Hive} conf via + * {@link org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl} + */ + @Test + public void testExplainDdlAcidTableAuthorized() throws Exception { + HiveConf queryConf = new HiveConf(hiveConf); + HiveConf.setBoolVar(queryConf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED, true); + runExplainDdl(queryConf); + } + + private void runExplainDdl(HiveConf queryConf) throws Exception { + Driver driver = new Driver(new QueryState.Builder().withHiveConf(queryConf).build(), null); + driver.setMaxRows(10000); + try { + driver.run(EXPLAIN_DDL); + } finally { + driver.close(); + driver.destroy(); + } + } +}