diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java index 4b0e75fafb51..f067a94830de 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -111,4 +111,6 @@ public class Constants { public static final String CLUSTER_ID_CLI_OPT_NAME = "hive.cluster.id"; public static final String CLUSTER_ID_HIVE_CONF_PROP = "hive.cluster.id"; public static final String ICEBERG_PARTITION_COLUMNS = "partition,spec_id"; + + public static final String NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM = "hive.storage.native.view.handler"; } diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 57f7cadac181..de96936df4c2 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -445,6 +445,8 @@ public enum ErrorMsg { @Deprecated // kept for backwards reference REPLACE_VIEW_WITH_MATERIALIZED(10400, "Attempt to replace view {0} with materialized view", true), REPLACE_MATERIALIZED_WITH_VIEW(10401, "Attempt to replace materialized view {0} with view", true), + VIEW_STORAGE_HANDLER_UNSUPPORTED(10448, "CREATE VIEW only supports STORED BY ICEBERG for native " + + "Iceberg views; unsupported storage clause: {0}", true), UPDATE_DELETE_VIEW(10402, "You cannot update or delete records in a view"), MATERIALIZED_VIEW_DEF_EMPTY(10403, "Query for the materialized view rebuild could not be retrieved"), MERGE_PREDIACTE_REQUIRED(10404, "MERGE statement with both UPDATE and DELETE clauses " + diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java index c0c959974a61..10f6736c3538 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java @@ -302,7 +302,7 @@ private Table newHMSView(ViewMetadata metadata) { tableType().name()); } - private String sqlFor(ViewMetadata metadata) { + public static String sqlFor(ViewMetadata metadata) { SQLViewRepresentation closest = null; for (ViewRepresentation representation : metadata.currentVersion().representations()) { if (representation instanceof SQLViewRepresentation) { diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index 95e1e5b36623..7330d5ba863b 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -46,6 +46,11 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.SQLViewRepresentation; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewMetadata; import org.apache.thrift.TException; public class MetastoreUtil { @@ -148,6 +153,74 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co return result; } + /** + * Builds a Hive metastore {@link Table} representation for an Iceberg {@link View}, for clients + * (e.g. {@code HiveRESTCatalogClient}) that bridge Iceberg catalog metadata into the HMS API. + */ + public static Table toHiveView(View view, Configuration conf) { + Table result = new Table(); + TableName tableName = + TableName.fromString( + view.name(), MetaStoreUtils.getDefaultCatalog(conf), Warehouse.DEFAULT_DATABASE_NAME); + result.setCatName(tableName.getCat()); + result.setDbName(tableName.getDb()); + result.setTableName(tableName.getTable()); + result.setTableType(TableType.VIRTUAL_VIEW.toString()); + + ViewMetadata metadata = ((BaseView) view).operations().current(); + String sqlText = viewSqlText(view, metadata); + result.setViewOriginalText(sqlText); + result.setViewExpandedText(sqlText); + + long nowMillis = System.currentTimeMillis(); + int nowSec = (int) (nowMillis / 1000); + String owner = + PropertyUtil.propertyAsString( + metadata.properties(), HiveCatalog.HMS_TABLE_OWNER, System.getProperty("user.name")); + result.setOwner(owner); + result.setCreateTime(nowSec); + result.setLastAccessTime(nowSec); + result.setRetention(Integer.MAX_VALUE); + + boolean hiveEngineEnabled = false; + result.setSd(HiveOperationsBase.storageDescriptor(metadata.schema(), metadata.location(), hiveEngineEnabled)); + StorageDescriptor sd = result.getSd(); + + if (sd.getBucketCols() == null) { + sd.setBucketCols(Lists.newArrayList()); + } + + if (sd.getSortCols() == null) { + sd.setSortCols(Lists.newArrayList()); + } + + long maxHiveTablePropertySize = + conf.getLong( + HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE, + HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT); + HMSTablePropertyHelper.updateHmsTableForIcebergView( + metadata.metadataFileLocation(), + result, + metadata, + Collections.emptySet(), + maxHiveTablePropertySize, + null); + + String catalogType = IcebergCatalogProperties.getCatalogType(conf); + if (!StringUtils.isEmpty(catalogType) && !IcebergCatalogProperties.NO_CATALOG_TYPE.equals(catalogType)) { + result.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.getCatalogType(conf)); + } + return result; + } + + private static String viewSqlText(View view, ViewMetadata metadata) { + SQLViewRepresentation hiveRepr = view.sqlFor("hive"); + if (hiveRepr != null) { + return hiveRepr.sql(); + } + return HiveViewOperations.sqlFor(metadata); + } + private static StorageDescriptor getHiveStorageDescriptor(org.apache.iceberg.Table table) { var result = new StorageDescriptor(); result.setCols(HiveSchemaUtil.convert(table.schema())); diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java new file mode 100644 index 000000000000..c4e45499a62c --- /dev/null +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/NativeIcebergViewSupport.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.hive; + +import java.io.Closeable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.view.ViewBuilder; + +/** + * Commits a native Iceberg view through the configured default Iceberg catalog (HiveCatalog or REST + * catalog, etc.) when {@code Catalog} also implements {@link ViewCatalog}. + */ +public final class NativeIcebergViewSupport { + + /** Value stored with {@link Constants#NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM} for Iceberg native views. */ + public static final String NATIVE_ICEBERG_VIEW_HANDLER_FQCN = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; + + /** + * HMS / Iceberg view marker entries for a native Iceberg catalog view (same map as + * {@code HiveIcebergStorageHandler#getNativeViewHmsTableProperties()}). + */ + public static Map defaultNativeViewMarkerTableProperties() { + return Map.of(Constants.NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM, NATIVE_ICEBERG_VIEW_HANDLER_FQCN); + } + + private NativeIcebergViewSupport() { + } + + /** + * Creates or replaces a view in the Iceberg catalog. + * + * @return {@code false} if skipped because {@code ifNotExists} is true and the view already exists + */ + public static boolean createOrReplaceNativeView(Configuration conf, String databaseName, String viewName, + List fieldSchemas, String viewSql, Map tblProperties, String comment, + boolean replace, boolean ifNotExists) throws Exception { + + TableIdentifier identifier = TableIdentifier.of(databaseName, viewName); + String catalogName = IcebergCatalogProperties.getCatalogName(conf); + Map catalogProps = IcebergCatalogProperties.getCatalogProperties(conf, catalogName); + Catalog catalog = CatalogUtil.buildIcebergCatalog(catalogName, catalogProps, conf); + try { + ViewCatalog viewCatalog = asViewCatalog(catalog, catalogName); + if (!replace && ifNotExists && viewCatalog.viewExists(identifier)) { + return false; + } + + Map mergedProps = mergeDefaultNativeViewTableProperties(tblProperties); + ViewBuilder builder = + viewCatalog + .buildView(identifier) + .withSchema(HiveSchemaUtil.convert(fieldSchemas, Collections.emptyMap(), true)) + .withDefaultNamespace(Namespace.of(identifier.namespace().level(0))) + .withQuery("hive", viewSql); + if (StringUtils.isNotBlank(comment)) { + builder = builder.withProperty("comment", comment); + } + for (Map.Entry e : mergedProps.entrySet()) { + if (e.getKey() != null && e.getValue() != null) { + builder = builder.withProperty(e.getKey(), e.getValue()); + } + } + if (replace) { + builder.createOrReplace(); + } else { + builder.create(); + } + return true; + } finally { + if (catalog instanceof Closeable) { + ((Closeable) catalog).close(); + } + } + } + + private static ViewCatalog asViewCatalog(Catalog catalog, String catalogName) { + if (!(catalog instanceof ViewCatalog)) { + throw new UnsupportedOperationException( + String.format( + "Iceberg catalog '%s' does not implement ViewCatalog.", + catalogName) + + " Native views require a catalog that implements ViewCatalog (e.g. HiveCatalog or REST)."); + } + return (ViewCatalog) catalog; + } + + /** + * Fills Iceberg native-view HMS / view marker properties when absent (e.g. direct catalog callers). + * Handlers that delegate here after {@code HiveStorageHandler#getNativeViewHmsTableProperties()} already + * supplied markers get the same result. + */ + public static Map mergeDefaultNativeViewTableProperties(Map tblProperties) { + Map merged = Maps.newHashMap(); + if (tblProperties != null) { + merged.putAll(tblProperties); + } + for (Map.Entry e : defaultNativeViewMarkerTableProperties().entrySet()) { + merged.putIfAbsent(e.getKey(), e.getValue()); + } + return merged; + } +} diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java index 4390d5a0bca1..2194b9950e3e 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java @@ -21,10 +21,12 @@ import java.io.IOException; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Set; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; @@ -43,7 +45,9 @@ import org.apache.iceberg.SortOrder; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.hive.HMSTablePropertyHelper; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; @@ -54,6 +58,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.view.View; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,10 +128,19 @@ public List getTables(String catName, String dbName, String tablePattern Pattern pattern = Pattern.compile(regex); // List tables from the specific database (namespace) and filter them. - return restCatalog.listTables(Namespace.of(dbName)).stream() + Set names = new LinkedHashSet<>(); + restCatalog.listTables(Namespace.of(dbName)).stream() .map(TableIdentifier::name) .filter(pattern.asPredicate()) - .toList(); + .forEach(names::add); + if (restCatalog instanceof ViewCatalog) { + ((ViewCatalog) restCatalog) + .listViews(Namespace.of(dbName)).stream() + .map(TableIdentifier::name) + .filter(pattern.asPredicate()) + .forEach(names::add); + } + return Lists.newArrayList(names); } @Override @@ -136,7 +150,12 @@ public List getAllTables(String catName, String dbName) { @Override public void dropTable(Table table, boolean deleteData, boolean ignoreUnknownTab, boolean ifPurge) throws TException { - restCatalog.dropTable(TableIdentifier.of(table.getDbName(), table.getTableName())); + TableIdentifier id = TableIdentifier.of(table.getDbName(), table.getTableName()); + if (restCatalog instanceof ViewCatalog && ((ViewCatalog) restCatalog).viewExists(id)) { + ((ViewCatalog) restCatalog).dropView(id); + } else { + restCatalog.dropTable(id); + } } private void validateCurrentCatalog(String catName) { @@ -149,7 +168,11 @@ private void validateCurrentCatalog(String catName) { @Override public boolean tableExists(String catName, String dbName, String tableName) { validateCurrentCatalog(catName); - return restCatalog.tableExists(TableIdentifier.of(dbName, tableName)); + TableIdentifier id = TableIdentifier.of(dbName, tableName); + if (restCatalog.tableExists(id)) { + return true; + } + return restCatalog instanceof ViewCatalog && ((ViewCatalog) restCatalog).viewExists(id); } @Override @@ -178,14 +201,22 @@ public Database getDatabase(String catName, String dbName) throws NoSuchObjectEx @Override public Table getTable(GetTableRequest tableRequest) throws TException { validateCurrentCatalog(tableRequest.getCatName()); - org.apache.iceberg.Table icebergTable; + TableIdentifier id = + TableIdentifier.of(tableRequest.getDbName(), tableRequest.getTblName()); try { - icebergTable = restCatalog.loadTable(TableIdentifier.of(tableRequest.getDbName(), - tableRequest.getTblName())); - } catch (NoSuchTableException exception) { + org.apache.iceberg.Table icebergTable = restCatalog.loadTable(id); + return MetastoreUtil.toHiveTable(icebergTable, conf); + } catch (NoSuchTableException tableMissing) { + if (restCatalog instanceof ViewCatalog) { + try { + View icebergView = ((ViewCatalog) restCatalog).loadView(id); + return MetastoreUtil.toHiveView(icebergView, conf); + } catch (NoSuchViewException viewMissing) { + throw new NoSuchObjectException(); + } + } throw new NoSuchObjectException(); } - return MetastoreUtil.toHiveTable(icebergTable, conf); } @Override diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java new file mode 100644 index 000000000000..3d05f113febb --- /dev/null +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestNativeIcebergViewSupport.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.hive; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.View; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE; +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestNativeIcebergViewSupport { + + private static final String DB = "native_vw_db"; + private static final String VIEW = "native_vw"; + + @RegisterExtension + private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION = + HiveMetastoreExtension.builder().withDatabase(DB).build(); + + @AfterEach + public void dropView() { + HiveCatalog cat = verifyCatalog(); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + cat.dropView(id); + } + + private HiveConf nativeViewConf() { + HiveConf conf = new HiveConf(HIVE_METASTORE_EXTENSION.hiveConf()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT, "hive"); + conf.set( + IcebergCatalogProperties.catalogPropertyConfigKey("hive", ICEBERG_CATALOG_TYPE), + ICEBERG_CATALOG_TYPE_HIVE); + return conf; + } + + private HiveCatalog verifyCatalog() { + return (HiveCatalog) + CatalogUtil.loadCatalog( + HiveCatalog.class.getName(), + ICEBERG_CATALOG_TYPE_HIVE, + ImmutableMap.of( + CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS, + String.valueOf(TimeUnit.SECONDS.toMillis(10))), + HIVE_METASTORE_EXTENSION.hiveConf()); + } + + @Test + public void testCreateCommitsNativeViewWithMarkerProperty() throws Exception { + HiveConf conf = nativeViewConf(); + List cols = + Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)); + String sql = String.format("select id, name from %s.src_tbl", DB); + Map props = Collections.singletonMap("k1", "v1"); + + boolean created = + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, sql, props, "hello-view", false, false); + assertThat(created).isTrue(); + + HiveCatalog cat = verifyCatalog(); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + assertThat(cat.viewExists(id)).isTrue(); + View view = cat.loadView(id); + assertThat( + view.properties().get(Constants.NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM)) + .isEqualTo(NativeIcebergViewSupport.NATIVE_ICEBERG_VIEW_HANDLER_FQCN); + assertThat(view.properties().get("comment")).isEqualTo("hello-view"); + assertThat(view.properties().get("k1")).isEqualTo("v1"); + HiveViewOperations ops = (HiveViewOperations) ((BaseView) view).operations(); + assertThat(ops.current().currentVersion().representations()).isNotEmpty(); + } + + @Test + public void testCreateOrReplaceNativeViewSkipsWhenViewExistsAndIfNotExistsFlagTrue() throws Exception { + HiveConf conf = nativeViewConf(); + List cols = Collections.singletonList(new FieldSchema("id", "int", null)); + + assertThat( + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 1 as id", null, null, false, false)) + .isTrue(); + assertThat( + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 2 as id", null, null, false, true)) + .isFalse(); + } + + @Test + public void testCreateOrReplaceNativeViewReplacesExistingWhenReplaceFlagTrue() throws Exception { + HiveConf conf = nativeViewConf(); + List cols = Collections.singletonList(new FieldSchema("id", "int", null)); + + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 1 as id", null, null, false, false); + assertThat( + NativeIcebergViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 2 as id", null, null, true, false)) + .isTrue(); + + assertThat(verifyCatalog().viewExists(TableIdentifier.of(DB, VIEW))).isTrue(); + } +} diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 65852f1a8553..06a0a97c4e4e 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -88,6 +88,7 @@ import org.apache.hadoop.hive.ql.io.parquet.vector.VectorizedParquetRecordReader; import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.metadata.CreateNativeViewRequest; import org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -183,6 +184,7 @@ import org.apache.iceberg.hive.HiveTableOperations; import org.apache.iceberg.hive.IcebergCatalogProperties; import org.apache.iceberg.hive.MetastoreUtil; +import org.apache.iceberg.hive.NativeIcebergViewSupport; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; @@ -383,6 +385,35 @@ public boolean supportsPartitioning() { return true; } + @Override + public boolean supportsNativeViewCatalog() { + return true; + } + + @Override + public Map getNativeViewHmsTableProperties() { + return NativeIcebergViewSupport.defaultNativeViewMarkerTableProperties(); + } + + @Override + public boolean createOrReplaceNativeView(Configuration conf, CreateNativeViewRequest request) + throws HiveException { + try { + return NativeIcebergViewSupport.createOrReplaceNativeView( + conf, + request.getDatabaseName(), + request.getViewName(), + request.getSchema(), + request.getExpandedText(), + request.getProperties(), + request.getComment(), + request.isReplace(), + request.isIfNotExists()); + } catch (Exception e) { + throw new HiveException(e); + } + } + /** * @param jobConf Job configuration for InputFormat to access * @param deserializer Deserializer diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q new file mode 100644 index 000000000000..3241dbe0175c --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q @@ -0,0 +1,55 @@ +-- SORT_QUERY_RESULTS +-- Mask random uuid +--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ + +create database ice_native_view_db; +use ice_native_view_db; + +create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc; + +INSERT INTO src_ice VALUES + ('fn1','ln1', 1), + ('fn2','ln2', 1), + ('fn3','ln3', 1), + ('fn4','ln4', 1), + ('fn5','ln5', 2), + ('fn6','ln6', 2), + ('fn7','ln7', 2); + +---------------------------------------------------------------- +-- Iceberg native view with STORED BY ICEBERG before AS +---------------------------------------------------------------- + +create view v_ice stored by iceberg as select * from src_ice; +select * from v_ice; +desc formatted v_ice; +drop view v_ice; + +---------------------------------------------------------------- +-- Iceberg native view from when STORED BY ICEBERG is omitted and Hive conf +-- 'hive.default.storage.handler.class' is 'HiveIcebergStorageHandler' +---------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view v_def as select * from src_ice; +select first_name, dept_id from v_def; +desc formatted v_def; +drop view v_def; + +---------------------------------------------------------------- +-- Regular Hive view when STORED BY ICEBERG is omitted and Hive conf +-- 'hive.default.storage.handler.class' is undefined +---------------------------------------------------------------- + +set hive.default.storage.handler.class=; + +create view v_hive as select * from src_ice; +select * from v_hive; +desc formatted v_hive; +drop view v_hive; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q index 81982ca44d98..203ed0dfa522 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q @@ -15,8 +15,6 @@ --! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+refused\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --- Mask compaction id as they will be allocated in parallel threads ---! qt:replace:/^[0-9]/#Masked#/ -- Mask removed file size --! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ -- Mask iceberg version @@ -47,7 +45,7 @@ partitioned by (company_id bigint) stored by iceberg stored as orc; ----------------------------------------------------------------------------- ---! Creating table with a valid catalog name in table properties +--! Creating a table with a valid catalog name in table properties ----------------------------------------------------------------------------- create table ice_orc2 ( @@ -74,6 +72,55 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; +--------------------------------------------------------------------------------------------------------------------- +--! Iceberg Native View tests +--------------------------------------------------------------------------------------------------------------------- + +-- Enable once CBO supports Iceberg native views on partitioned tables +set hive.cbo.enable=false; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! without a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v1 stored by iceberg as select * from ice_orc1; +select * from ice_v1; +desc formatted ice_v1; +drop view ice_v1; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v2 stored by iceberg as select * from ice_orc2; +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table without catalog name in table properties +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v3 as select * from ice_orc1; +select * from ice_v3; +desc formatted ice_v3; +drop view ice_v3; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v4 as select * from ice_orc2; +select * from ice_v4; +desc formatted ice_v4; +drop view ice_v4; + ----------------------------------------------------------------------------- show tables; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q index 27f23122240b..4c1c8426e745 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q @@ -15,8 +15,6 @@ --! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+refused\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --- Mask compaction id as they will be allocated in parallel threads ---! qt:replace:/^[0-9]/#Masked#/ -- Mask removed file size --! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ -- Mask iceberg version @@ -47,7 +45,7 @@ partitioned by (company_id bigint) stored by iceberg stored as orc; ----------------------------------------------------------------------------- ---! Creating table with a valid catalog name in table properties +--! Creating a table with a valid catalog name in table properties ----------------------------------------------------------------------------- create table ice_orc2 ( @@ -69,6 +67,55 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; +--------------------------------------------------------------------------------------------------------------------- +--! Iceberg Native View tests +--------------------------------------------------------------------------------------------------------------------- + +-- Enable once CBO supports Iceberg native views on partitioned tables +set hive.cbo.enable=false; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! without a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v1 stored by iceberg as select * from ice_orc1; +select * from ice_v1; +desc formatted ice_v1; +drop view ice_v1; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view (with STORED BY ICEBERG in the SQL command) on a REST catalog table +--! with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v2 stored by iceberg as select * from ice_orc2; +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table without catalog name in table properties +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v3 as select * from ice_orc1; +select * from ice_v3; +desc formatted ice_v3; +drop view ice_v3; + +----------------------------------------------------------------------------------------------- +--! Native view when STORED BY is omitted and 'hive.default.storage.handler.class' Hive conf +--! is set to 'HiveIcebergStorageHandler' on a REST catalog table with a catalog name in table properties +----------------------------------------------------------------------------------------------- + +create view ice_v4 as select * from ice_orc2; +select * from ice_v4; +desc formatted ice_v4; +drop view ice_v4; + ----------------------------------------------------------------------------- show tables; diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out new file mode 100644 index 000000000000..66e871c0b5c7 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_native_view.q.out @@ -0,0 +1,270 @@ +PREHOOK: query: create database ice_native_view_db +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:ice_native_view_db +POSTHOOK: query: create database ice_native_view_db +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:ice_native_view_db +PREHOOK: query: use ice_native_view_db +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:ice_native_view_db +POSTHOOK: query: use ice_native_view_db +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:ice_native_view_db +PREHOOK: query: create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: INSERT INTO src_ice VALUES + ('fn1','ln1', 1), + ('fn2','ln2', 1), + ('fn3','ln3', 1), + ('fn4','ln4', 1), + ('fn5','ln5', 2), + ('fn6','ln6', 2), + ('fn7','ln7', 2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: INSERT INTO src_ice VALUES + ('fn1','ln1', 1), + ('fn2','ln2', 1), + ('fn3','ln3', 1), + ('fn4','ln4', 1), + ('fn5','ln5', 2), + ('fn6','ln6', 2), + ('fn7','ln7', 2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: create view v_ice stored by iceberg as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: create view v_ice stored by iceberg as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: Lineage: v_ice.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_ice.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_ice.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_ice +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_ice +#### A masked pattern was here #### +POSTHOOK: query: select * from v_ice +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_ice +#### A masked pattern was here #### +fn1 ln1 1 +fn2 ln2 1 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: desc formatted v_ice +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: query: desc formatted v_ice +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_ice +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_ice +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: drop view v_ice +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: ice_native_view_db@v_ice +PREHOOK: query: create view v_def as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: create view v_def as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_def +POSTHOOK: Lineage: v_def.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_def.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_def.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select first_name, dept_id from v_def +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_def +#### A masked pattern was here #### +POSTHOOK: query: select first_name, dept_id from v_def +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_def +#### A masked pattern was here #### +fn1 1 +fn2 1 +fn3 1 +fn4 1 +fn5 2 +fn6 2 +fn7 2 +PREHOOK: query: desc formatted v_def +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_def +POSTHOOK: query: desc formatted v_def +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_def +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_def +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: drop view v_def +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: ice_native_view_db@v_def +PREHOOK: query: create view v_hive as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: create view v_hive as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: Lineage: v_hive.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_hive.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_hive.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_hive +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_hive +#### A masked pattern was here #### +POSTHOOK: query: select * from v_hive +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_hive +#### A masked pattern was here #### +fn1 ln1 1 +fn2 ln2 1 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: desc formatted v_hive +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: query: desc formatted v_hive +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_hive +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select * from src_ice +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_hive +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_hive +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: drop view v_hive +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: Output: ice_native_view_db@v_hive diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 8bba659e8fd1..2e2522dd1c22 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -180,6 +180,296 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 +PREHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +POSTHOOK: Lineage: ice_v1.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v1 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: drop view ice_v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v1 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v2 +POSTHOOK: Lineage: ice_v2.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v2.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v2 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v2 +POSTHOOK: query: desc formatted ice_v2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v2 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v2 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: drop view ice_v2 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Output: ice_rest@ice_v2 +PREHOOK: query: create view ice_v3 as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: create view ice_v3 as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v3 +POSTHOOK: Lineage: ice_v3.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v3.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v3.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v3 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v3 +POSTHOOK: query: desc formatted ice_v3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v3 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v3 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v3 +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: drop view ice_v3 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v3 +POSTHOOK: Output: ice_rest@ice_v3 +PREHOOK: query: create view ice_v4 as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: create view ice_v4 as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v4 +POSTHOOK: Lineage: ice_v4.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v4.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v4.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v4 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v4 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v4 +POSTHOOK: query: desc formatted ice_v4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v4 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v4 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v4 +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: drop view ice_v4 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v4 +POSTHOOK: Output: ice_rest@ice_v4 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 409eb484480b..c1e417a90836 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -180,6 +180,296 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 +PREHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view ice_v1 stored by iceberg as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +POSTHOOK: Lineage: ice_v1.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v1 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: drop view ice_v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v1 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: create view ice_v2 stored by iceberg as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v2 +POSTHOOK: Lineage: ice_v2.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v2.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v2 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v2 +POSTHOOK: query: desc formatted ice_v2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v2 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v2 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: drop view ice_v2 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Output: ice_rest@ice_v2 +PREHOOK: query: create view ice_v3 as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: create view ice_v3 as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v3 +POSTHOOK: Lineage: ice_v3.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v3.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v3.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v3 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v3 +POSTHOOK: query: desc formatted ice_v3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v3 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v3 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v3 +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: drop view ice_v3 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v3 +POSTHOOK: Output: ice_rest@ice_v3 +PREHOOK: query: create view ice_v4 as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: create view ice_v4 as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v4 +POSTHOOK: Lineage: ice_v4.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:Transform: identity), ] +POSTHOOK: Lineage: ice_v4.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v4.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v4 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v4 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v4 +POSTHOOK: query: desc formatted ice_v4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v4 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + hive.storage.native.view.handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler +#### A masked pattern was here #### + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v4 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v4 +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: drop view ice_v4 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v4 +POSTHOOK: Output: ice_rest@ice_v4 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest diff --git a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java index fd30223c9934..6ba5ef472e2f 100644 --- a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java +++ b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; @@ -38,13 +39,15 @@ import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.hive.IcebergCatalogProperties; import org.apache.iceberg.hive.HiveSchemaUtil; +import org.apache.iceberg.hive.IcebergCatalogProperties; +import org.apache.iceberg.hive.NativeIcebergViewSupport; import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -67,11 +70,14 @@ public abstract class TestHiveRESTCatalogClientITBase { static final String DB_NAME = "ice_db"; + static final String VIEW_DB_NAME = "ice_db_view"; + static final String NATIVE_VIEW_NAME = "native_rest_v"; static final String TABLE_NAME = "ice_tbl"; static final String CATALOG_NAME = "ice01"; static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; static final String REST_CATALOG_PREFIX = String.format("%s%s.", IcebergCatalogProperties.CATALOG_CONFIG_PREFIX, CATALOG_NAME); + static final String ICEBERG_VIEW_TYPE_VALUE = "iceberg-view"; HiveConf hiveConf; Configuration conf; @@ -195,6 +201,50 @@ public void testIceberg() throws Exception { Assertions.assertFalse(msClient.getAllDatabases(CATALOG_NAME).contains(DB_NAME)); } + @Test + public void testNativeIcebergView() throws Exception { + Database db = new Database(); + db.setCatalogName(CATALOG_NAME); + db.setName(VIEW_DB_NAME); + db.setOwnerType(PrincipalType.USER); + db.setOwnerName(System.getProperty("user.name")); + String warehouseDir = MetastoreConf.get(conf, MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname()); + db.setLocationUri(warehouseDir + "/" + VIEW_DB_NAME + ".db"); + hive.createDatabase(db, true); + + List cols = Collections.singletonList(new FieldSchema("x", "int", "")); + NativeIcebergViewSupport.createOrReplaceNativeView( + hiveConf, + VIEW_DB_NAME, + NATIVE_VIEW_NAME, + cols, + "select 1 as x", + null, + "rest-native-view", + false, + false); + + GetTableRequest getTableRequest = new GetTableRequest(); + getTableRequest.setCatName(CATALOG_NAME); + getTableRequest.setDbName(VIEW_DB_NAME); + getTableRequest.setTblName(NATIVE_VIEW_NAME); + Table view = msClient.getTable(getTableRequest); + Assertions.assertEquals(TableType.VIRTUAL_VIEW.name(), view.getTableType()); + String tableTypeProp = view.getParameters().get(BaseMetastoreTableOperations.TABLE_TYPE_PROP); + Assertions.assertNotNull(tableTypeProp); + Assertions.assertEquals(ICEBERG_VIEW_TYPE_VALUE, tableTypeProp.toLowerCase()); + + Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); + + List names = msClient.getTables(CATALOG_NAME, VIEW_DB_NAME, "*"); + Assertions.assertTrue(names.contains(NATIVE_VIEW_NAME)); + + msClient.dropTable(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME); + Assertions.assertFalse(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); + + msClient.dropDatabase(VIEW_DB_NAME); + } + private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName, Map tableParameters) throws Exception { db.dropTable(catName, dbName, tableName); diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 64a761acfddb..fd38e9c1f2ab 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -433,6 +433,7 @@ iceberg.llap.query.files=\ iceberg_create_locally_zordered_table.q,\ iceberg_merge_delete_files.q,\ iceberg_merge_files.q,\ + iceberg_native_view.q,\ llap_iceberg_read_orc.q,\ llap_iceberg_read_parquet.q,\ puffin_col_stats_with_time_travel.q,\ @@ -484,6 +485,7 @@ iceberg.llap.only.query.files=\ iceberg_create_locally_zordered_table.q,\ iceberg_merge_delete_files.q,\ iceberg_merge_files.q,\ + iceberg_native_view.q,\ llap_iceberg_read_orc.q,\ llap_iceberg_read_parquet.q,\ puffin_col_stats_with_time_travel.q diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java index 6b1680d2f232..b9b556b6c397 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java @@ -35,10 +35,10 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QTestHiveRoot; import org.apache.hadoop.hive.ql.QTestSystemProperties; import org.apache.hadoop.hive.ql.QTestMiniClusters.FsType; import org.apache.hadoop.hive.ql.QTestMiniClusters.MiniClusterType; -import org.apache.hive.testutils.HiveTestEnvSetup; import com.google.common.base.Splitter; import org.slf4j.Logger; @@ -46,7 +46,7 @@ public abstract class AbstractCliConfig { - public static final String HIVE_ROOT = HiveTestEnvSetup.HIVE_ROOT; + public static final String HIVE_ROOT = QTestHiveRoot.HIVE_ROOT; private static final Logger LOG = LoggerFactory.getLogger(AbstractCliConfig.class); private String queryFile; diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java index 0d343545e4d9..44313787f2f4 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestMiniClusters.MiniClusterType; import org.apache.hadoop.hive.ql.processors.CommandProcessorException; -import org.apache.hive.testutils.HiveTestEnvSetup; +import org.apache.hadoop.hive.ql.QTestHiveRoot; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -160,8 +160,8 @@ public Map getHiveVariable() { } }).substitute(new HiveConf(), qt.getConf().get(HCONF_TEST_BLOBSTORE_PATH)); - testBlobstorePath = HiveTestEnvSetup.ensurePathEndsInSlash(testBlobstorePath); - testBlobstorePath += HiveTestEnvSetup.ensurePathEndsInSlash(this.getClass().getSimpleName()); // name of child class + testBlobstorePath = QTestHiveRoot.ensurePathEndsInSlash(testBlobstorePath); + testBlobstorePath += QTestHiveRoot.ensurePathEndsInSlash(this.getClass().getSimpleName()); // name of child class String uid = new SimpleDateFormat("yyyyMMdd.HHmmss.SSS").format(Calendar.getInstance().getTime()) + "-" + String.format("%03d", (int)(Math.random() * 999)); testBlobstorePathUnique = testBlobstorePath + uid; diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestHiveRoot.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestHiveRoot.java new file mode 100644 index 000000000000..5b63fe0301eb --- /dev/null +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestHiveRoot.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Locates the Hive source tree root for integration-test utilities in {@code hive-it-util}. + * Logic is aligned with {@code org.apache.hive.testutils.HiveTestEnvSetup#getHiveRoot()} so that + * {@code hive-it-util} main code does not depend on classes under {@code ql} test sources. + */ +public final class QTestHiveRoot { + + public static final String HIVE_ROOT = resolveHiveRoot(); + + private QTestHiveRoot() {} + + private static String resolveHiveRoot() { + List candidateSiblings = new ArrayList<>(); + if (System.getProperty("hive.root") != null) { + try { + candidateSiblings.add(new File(System.getProperty("hive.root")).getCanonicalPath()); + } catch (IOException e) { + throw new RuntimeException("error getting hive.root", e); + } + } + candidateSiblings.add(new File(".").getAbsolutePath()); + + for (String string : candidateSiblings) { + File curr = new File(string); + do { + String[] names = curr.list(); + if (names != null) { + Set lls = new HashSet<>(Arrays.asList(names)); + if (lls.contains("itests") && lls.contains("ql") && lls.contains("metastore")) { + System.out.println("detected hiveRoot: " + curr); + return ensurePathEndsInSlash(curr.getAbsolutePath()); + } + } + curr = curr.getParentFile(); + } while (curr != null); + } + throw new RuntimeException("unable to find hiveRoot"); + } + + public static String ensurePathEndsInSlash(String path) { + if (path == null) { + throw new NullPointerException("Path cannot be null"); + } + if (path.endsWith(File.separator)) { + return path; + } else { + return path + File.separator; + } + } +} diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestSysDbHandler.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestSysDbHandler.java index 708251e9403b..3074c0fd8eb4 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestSysDbHandler.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestSysDbHandler.java @@ -20,7 +20,7 @@ import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; import org.apache.hadoop.hive.ql.QTestUtil; -import org.apache.hive.testutils.HiveTestEnvSetup; +import org.apache.hadoop.hive.ql.QTestHiveRoot; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,7 +47,7 @@ public void beforeTest(QTestUtil qt) throws Exception { if (enabled) { String schemaVersion = MetaStoreSchemaInfoFactory.get(qt.getConf()).getHiveSchemaVersion(); String stsdbPath = - HiveTestEnvSetup.HIVE_ROOT + "/standalone-metastore/metastore-server/src/main/sql/hive/hive-schema-" + schemaVersion + ".hive.sql"; + QTestHiveRoot.HIVE_ROOT + "/standalone-metastore/metastore-server/src/main/sql/hive/hive-schema-" + schemaVersion + ".hive.sql"; qt.getCliDriver().processLine("source " + stsdbPath); qt.getCliDriver().processLine("use default"); } diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 0033aa3251c6..82b5e1e1e5a6 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -295,6 +295,7 @@ TOK_CREATEMACRO; TOK_DROPMACRO; TOK_TEMPORARY; TOK_CREATEVIEW; +TOK_VIEWMETADATAFORMAT; TOK_DROPVIEW; TOK_ALTERVIEW; TOK_ALTERVIEW_PROPERTIES; @@ -1651,6 +1652,7 @@ createViewStatement : KW_CREATE (orReplace)? KW_VIEW (ifNotExists)? name=tableName (LPAREN columnNameCommentList RPAREN)? tableComment? viewPartition? tablePropertiesPrefixed? + (viewMeta=viewMetadataFormat)? KW_AS selectStatementWithCTE -> ^(TOK_CREATEVIEW $name orReplace? @@ -1659,6 +1661,7 @@ createViewStatement tableComment? viewPartition? tablePropertiesPrefixed? + $viewMeta? selectStatementWithCTE ) ; @@ -2054,6 +2057,14 @@ tableFileFormat -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; +// Native view metadata: only STORED BY (e.g. ICEBERG). No SerDe props or STORED AS tail. +viewMetadataFormat +@init { pushMsg("view metadata format", state); } +@after { popMsg(state); } + : KW_STORED KW_BY handlerId=identifier + -> ^(TOK_VIEWMETADATAFORMAT $handlerId) + ; + tableLocation @init { pushMsg("table location specification", state); } @after { popMsg(state); } diff --git a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java index 45a3a9e74180..9e0305bc3e8f 100644 --- a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java +++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseDefault.java @@ -90,6 +90,15 @@ public void testParseStructNamedDefault() throws Exception { assertFalse(tree.dump(), tree.toStringTree().contains("tok_default_value")); } + @Test + public void testParseCreateViewStoredByIceberg() throws Exception { + ASTNode tree = parseDriver.parse( + "create view v1 stored by iceberg as select * from t", null).getTree(); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_createview")); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_viewmetadataformat")); + assertTrue(tree.dump(), tree.toStringTree().contains("iceberg")); + } + @Test public void testParseStructFieldNamedDefault() throws Exception { ASTNode tree = parseDriver.parse( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java index 60b123dbb7bd..6ee260ff7cd8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java @@ -24,7 +24,9 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -34,6 +36,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -41,12 +44,14 @@ import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.StorageFormat; /** * Analyzer for create view commands. */ @DDLType(types = HiveParser.TOK_CREATEVIEW) public class CreateViewAnalyzer extends AbstractCreateViewAnalyzer { + public CreateViewAnalyzer(QueryState queryState) throws SemanticException { super(queryState); } @@ -75,6 +80,9 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List partitionColumnNames = children.containsKey(HiveParser.TOK_VIEWPARTCOLS) ? getColumnNames((ASTNode) children.remove(HiveParser.TOK_VIEWPARTCOLS).getChild(0)) : null; + ASTNode viewMetadataFormat = children.remove(HiveParser.TOK_VIEWMETADATAFORMAT); + String nativeViewStorageHandlerClass = resolveNativeViewStorageHandlerClass(viewMetadataFormat); + assert children.isEmpty(); if (ifNotExists && orReplace) { @@ -94,7 +102,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List partitionColumns = getPartitionColumns(partitionColumnNames); setColumnAccessInfo(analyzer.getColumnAccessInfo()); CreateViewDesc desc = new CreateViewDesc(fqViewName, schema, comment, properties, partitionColumnNames, - ifNotExists, orReplace, originalText, expandedText, partitionColumns); + ifNotExists, orReplace, originalText, expandedText, partitionColumns, nativeViewStorageHandlerClass); validateCreateView(desc, analyzer); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); @@ -191,6 +199,48 @@ private List getPartitionColumns(List partitionColumnNames) return partitionColumnsCopy; } + /** + * Returns the FQCN of the storage handler that should own native-catalog view metadata, or {@code null} for a + * classic HMS virtual view. Uses {@link HiveStorageHandler#supportsNativeViewCatalog()} on the resolved handler. + */ + private String resolveNativeViewStorageHandlerClass(ASTNode viewMetadataFormat) throws SemanticException { + String handlerClass; + if (viewMetadataFormat != null) { + if (viewMetadataFormat.getChildCount() != 1) { + throw new SemanticException("Internal error: expected single handler identifier in view metadata"); + } + String identifier = viewMetadataFormat.getChild(0).getText(); + handlerClass = StorageFormat.resolveStorageHandlerClassNameForView(identifier); + } else { + handlerClass = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER); + if (handlerClass != null) { + handlerClass = handlerClass.trim(); + if (handlerClass.isEmpty()) { + handlerClass = null; + } + } + } + if (StringUtils.isBlank(handlerClass)) { + return null; + } + try { + HiveStorageHandler handler = HiveUtils.getStorageHandler(conf, handlerClass); + if (handler != null && handler.supportsNativeViewCatalog()) { + return handlerClass; + } + } catch (HiveException e) { + if (viewMetadataFormat != null) { + throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg(e.getMessage()), e); + } + return null; + } + if (viewMetadataFormat != null) { + throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.getMsg( + "Native view metadata is not supported for storage handler: " + handlerClass)); + } + return null; + } + private void validateCreateView(CreateViewDesc desc, SemanticAnalyzer analyzer) throws SemanticException { try { validateTablesUsed(analyzer); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java index e71cbce773f1..9243d3fc6cfb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java @@ -40,6 +40,7 @@ public class CreateViewDesc extends AbstractCreateViewDesc { private final boolean ifNotExists; private final boolean replace; private final List partitionColumns; + private final String nativeViewStorageHandlerClass; private ReplicationSpec replicationSpec = null; private String ownerName = null; @@ -47,6 +48,13 @@ public class CreateViewDesc extends AbstractCreateViewDesc { public CreateViewDesc(String viewName, List schema, String comment, Map properties, List partitionColumnNames, boolean ifNotExists, boolean replace, String originalText, String expandedText, List partitionColumns) { + this(viewName, schema, comment, properties, partitionColumnNames, ifNotExists, replace, originalText, + expandedText, partitionColumns, null); + } + + public CreateViewDesc(String viewName, List schema, String comment, Map properties, + List partitionColumnNames, boolean ifNotExists, boolean replace, String originalText, + String expandedText, List partitionColumns, String nativeViewStorageHandlerClass) { super(viewName, schema, originalText, expandedText); this.comment = comment; this.properties = properties; @@ -54,6 +62,7 @@ public CreateViewDesc(String viewName, List schema, String comment, this.ifNotExists = ifNotExists; this.replace = replace; this.partitionColumns = partitionColumns; + this.nativeViewStorageHandlerClass = nativeViewStorageHandlerClass; } @Explain(displayName = "partition columns") @@ -89,6 +98,19 @@ public boolean isReplace() { return replace; } + /** + * @return FQCN of the {@link HiveStorageHandler} that stores view metadata in an external catalog, or + * {@code null} for a classic HMS-only virtual view. + */ + @Explain(displayName = "native view storage handler", displayOnlyOnTrue = true) + public String getNativeViewStorageHandlerClass() { + return nativeViewStorageHandlerClass; + } + + public boolean usesNativeViewCatalog() { + return nativeViewStorageHandlerClass != null && !nativeViewStorageHandlerClass.trim().isEmpty(); + } + /** * @param replicationSpec Sets the replication spec governing this create. * This parameter will have meaningful values only for creates happening as a result of a replication. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java index b6a41a8d4fb8..db10c27d4622 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.ddl.DDLOperation; @@ -29,11 +30,15 @@ import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.hooks.LineageInfo.DataContainer; +import org.apache.hadoop.hive.ql.metadata.CreateNativeViewRequest; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; +import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.HiveTableName; import org.apache.hadoop.hive.ql.parse.StorageFormat; +import java.util.HashMap; import java.util.Map; /** @@ -46,6 +51,11 @@ public CreateViewOperation(DDLOperationContext context, CreateViewDesc desc) { @Override public int execute() throws HiveException { + if (desc.usesNativeViewCatalog()) { + executeNativeCatalogView(); + return 0; + } + Table oldview = context.getDb().getTable(desc.getViewName(), false); if (oldview != null) { boolean isReplace = desc.isReplace(); @@ -82,6 +92,7 @@ public int execute() throws HiveException { if (desc.getProperties() != null) { oldview.getTTable().getParameters().putAll(desc.getProperties()); } + clearNativeViewCatalogMarkersIfPresent(oldview); oldview.setPartCols(desc.getPartitionColumns()); oldview.checkValidity(null); @@ -105,6 +116,91 @@ public int execute() throws HiveException { return 0; } + /** + * Creates a native-catalog view via the {@link HiveStorageHandler} selected at compile time. + */ + private void executeNativeCatalogView() throws HiveException { + Table oldView = context.getDb().getTable(desc.getViewName(), false); + + if (oldView != null) { + boolean isReplace = desc.isReplace(); + + if (desc.getReplicationSpec().isInReplicationScope()) { + Map dbParams = context.getDb().getDatabase(oldView.getDbName()).getParameters(); + if (desc.getReplicationSpec().allowEventReplacementInto(dbParams)) { + isReplace = true; + } else { + LOG.debug("DDLTask: Create native-catalog view is skipped as view {} is newer than update", + desc.getViewName()); + return; + } + } + + if (!isReplace) { + if (desc.getIfNotExists()) { + return; + } + throw new HiveException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(desc.getViewName())); + } + } + + TableName name = HiveTableName.of(desc.getViewName()); + boolean replace = oldView != null || desc.isReplace(); + + try { + HiveStorageHandler handler = + HiveUtils.getStorageHandler(context.getConf(), desc.getNativeViewStorageHandlerClass()); + Map viewProperties = new HashMap<>(); + if (desc.getProperties() != null) { + viewProperties.putAll(desc.getProperties()); + } + viewProperties.putAll(handler.getNativeViewHmsTableProperties()); + CreateNativeViewRequest request = + new CreateNativeViewRequest( + name.getDb(), + name.getTable(), + desc.getSchema(), + desc.getExpandedText(), + viewProperties, + desc.getComment(), + replace, + desc.getIfNotExists()); + boolean created = handler.createOrReplaceNativeView(context.getConf(), request); + if (!created) { + return; + } + } catch (HiveException e) { + Throwable cause = e.getCause(); + if (cause != null && cause.getClass().getName().endsWith("AlreadyExistsException")) { + throw new HiveException(ErrorMsg.TABLE_ALREADY_EXISTS.getMsg(desc.getViewName()), cause); + } + throw e; + } catch (Exception e) { + throw new HiveException(e); + } + + Table viewTbl = context.getDb().getTable(desc.getViewName(), false); + if (viewTbl != null) { + DDLUtils.addIfAbsentByName(new WriteEntity(viewTbl, WriteEntity.WriteType.DDL_NO_LOCK), + context.getWork().getOutputs()); + DataContainer dc = new DataContainer(viewTbl.getTTable()); + context.getQueryState().getLineageState().setLineage(new Path(desc.getViewName()), dc, viewTbl.getCols()); + } + } + + private void clearNativeViewCatalogMarkersIfPresent(Table oldview) throws HiveException { + Map params = oldview.getParameters(); + if (params == null) { + return; + } + String fqcn = params.get(Constants.NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM); + if (fqcn == null) { + return; + } + HiveStorageHandler handler = HiveUtils.getStorageHandler(context.getConf(), fqcn); + handler.clearNativeViewHmsTableProperties(params); + } + private Table createViewObject() throws HiveException { TableName name = HiveTableName.of(desc.getViewName()); Table view = new Table(name.getDb(), name.getTable()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateNativeViewRequest.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateNativeViewRequest.java new file mode 100644 index 000000000000..f45f697eccaa --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateNativeViewRequest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +/** + * Parameters for {@link HiveStorageHandler#createOrReplaceNativeView} (native catalog view DDL). + */ +public final class CreateNativeViewRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private final String databaseName; + private final String viewName; + private final List schema; + private final String expandedText; + private final Map properties; + private final String comment; + private final boolean replace; + private final boolean ifNotExists; + + public CreateNativeViewRequest( + String databaseName, + String viewName, + List schema, + String expandedText, + Map properties, + String comment, + boolean replace, + boolean ifNotExists) { + this.databaseName = databaseName; + this.viewName = viewName; + this.schema = schema; + this.expandedText = expandedText; + this.properties = properties == null ? null : Collections.unmodifiableMap(properties); + this.comment = comment; + this.replace = replace; + this.ifNotExists = ifNotExists; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getViewName() { + return viewName; + } + + public List getSchema() { + return schema; + } + + public String getExpandedText() { + return expandedText; + } + + public Map getProperties() { + return properties; + } + + public String getComment() { + return comment; + } + + public boolean isReplace() { + return replace; + } + + public boolean isIfNotExists() { + return ifNotExists; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 520c52a24a8f..1fcb42e8932b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -27,11 +27,13 @@ import java.util.concurrent.ExecutorService; import com.google.common.collect.Maps; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.common.type.SnapshotContext; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.api.AggrStats; @@ -1028,6 +1030,47 @@ default void setMergeTaskDeleteProperties(TableDesc tableDesc) { throw new UnsupportedOperationException("Storage handler does not support getting custom delete merge schema."); } + /** + * @return {@code true} if this handler may store CREATE VIEW text and column metadata in an external catalog + * (native view) rather than only as a classic HMS virtual view. + */ + default boolean supportsNativeViewCatalog() { + return false; + } + + /** + * HMS table-level parameters to set on the metastore stub for a native-catalog view (markers, etc.). + * Keys should be removed when {@linkplain #clearNativeViewHmsTableProperties(Map)} is invoked for the same + * handler class recorded under {@link Constants.NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM}. + */ + default Map getNativeViewHmsTableProperties() { + return Collections.emptyMap(); + } + + /** + * Removes entries added for native-catalog views (see {@link #getNativeViewHmsTableProperties()}). + */ + default void clearNativeViewHmsTableProperties(Map tableParameters) { + if (tableParameters == null) { + return; + } + for (String k : getNativeViewHmsTableProperties().keySet()) { + tableParameters.remove(k); + } + tableParameters.remove(Constants.NATIVE_VIEW_STORAGE_HANDLER_CLASS_PARAM); + } + + /** + * Creates or replaces a view in the external catalog backing this handler. + * + * @return {@code false} if the operation was skipped (e.g. IF NOT EXISTS and the view already exists) + */ + default boolean createOrReplaceNativeView(Configuration conf, CreateNativeViewRequest request) + throws HiveException { + throw new HiveException( + "createOrReplaceNativeView is not supported by storage handler " + getClass().getName()); + } + default boolean supportsDefaultColumnValues(Map tblProps) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java index 2472ad44ad00..23cab25c1966 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java @@ -89,6 +89,23 @@ public String outputFormat() { } } + /** + * Resolves {@code STORED BY identifier} for CREATE VIEW (short names such as {@code ICEBERG} or an FQCN). + */ + public static String resolveStorageHandlerClassNameForView(String identifierText) throws SemanticException { + if (StringUtils.isBlank(identifierText)) { + throw new SemanticException("Storage handler identifier in CREATE VIEW cannot be empty"); + } + String text = identifierText.trim(); + for (StorageHandlerTypes type : StorageHandlerTypes.NON_DEFAULT_TYPES) { + if (type.name().equalsIgnoreCase(text)) { + Objects.requireNonNull(type.className()); + return ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(type.className())); + } + } + return ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(text)); + } + public StorageFormat(Configuration conf) { this.conf = conf; this.serdeProps = new HashMap();