diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergFilterFactory.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergFilterFactory.java index d1d98b3a584e..cc15053bd7a5 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergFilterFactory.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergFilterFactory.java @@ -30,7 +30,6 @@ import java.time.temporal.ChronoUnit; import java.util.List; import java.util.stream.Collectors; -import org.apache.commons.lang3.ObjectUtils; import org.apache.hadoop.hive.ql.io.sarg.ExpressionTree; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; @@ -111,9 +110,7 @@ private static Expression translate(ExpressionTree tree, List lea */ private static Expression translateLeaf(PredicateLeaf leaf) { TransformSpec transformSpec = TransformSpec.fromStringWithColumnName(leaf.getColumnName()); - String columnName = transformSpec.getColumnName(); - UnboundTerm column = - ObjectUtils.defaultIfNull(toTerm(columnName, transformSpec), Expressions.ref(columnName)); + UnboundTerm column = SchemaUtils.toTerm(transformSpec); switch (leaf.getOperator()) { case EQUALS: @@ -144,30 +141,6 @@ private static Expression translateLeaf(PredicateLeaf leaf) { } } - public static UnboundTerm toTerm(String columnName, TransformSpec transformSpec) { - if (transformSpec == null) { - return null; - } - switch (transformSpec.getTransformType()) { - case YEAR: - return Expressions.year(columnName); - case MONTH: - return Expressions.month(columnName); - case DAY: - return Expressions.day(columnName); - case HOUR: - return Expressions.hour(columnName); - case TRUNCATE: - return Expressions.truncate(columnName, transformSpec.getTransformParam()); - case BUCKET: - return Expressions.bucket(columnName, transformSpec.getTransformParam()); - case IDENTITY: - return null; - default: - throw new UnsupportedOperationException("Unknown transformSpec: " + transformSpec); - } - } - // PredicateLeafImpl has a work-around for Kryo serialization with java.util.Date objects where it converts values to // Timestamp using Date#getTime. This conversion discards microseconds, so this is a necessary to avoid it. private static final DynFields.UnboundField LITERAL_FIELD = DynFields.builder() diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 64f81d35a902..865116414459 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -20,8 +20,6 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -35,7 +33,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ObjectUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -137,7 +134,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; -import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.Pair; @@ -574,56 +570,47 @@ public void rollbackAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTab @Override public void preTruncateTable(org.apache.hadoop.hive.metastore.api.Table table, EnvironmentContext context, - List partNames) - throws MetaException { + List partNames) throws MetaException { + this.tableProperties = IcebergTableProperties.getTableProperties(table, conf); this.icebergTable = Catalogs.loadTable(conf, tableProperties); - Map partitionFieldMap = icebergTable.spec().fields().stream() - .collect(Collectors.toMap(PartitionField::name, Function.identity())); - Expression finalExp = CollectionUtils.isEmpty(partNames) ? Expressions.alwaysTrue() : Expressions.alwaysFalse(); - if (partNames != null) { - for (String partName : partNames) { - Map specMap = Warehouse.makeSpecFromName(partName); - Expression subExp = Expressions.alwaysTrue(); - for (Map.Entry entry : specMap.entrySet()) { - // Since Iceberg encodes the values in UTF-8, we need to decode it. - String partColValue = URLDecoder.decode(entry.getValue(), StandardCharsets.UTF_8); - - if (partitionFieldMap.containsKey(entry.getKey())) { - PartitionField partitionField = partitionFieldMap.get(entry.getKey()); - Type resultType = partitionField.transform().getResultType(icebergTable.schema() - .findField(partitionField.sourceId()).type()); - TransformSpec.TransformType transformType = TransformSpec.fromString(partitionField.transform().toString()); - Object value = Conversions.fromPartitionString(resultType, partColValue); - Iterable iterable = () -> Collections.singletonList(value).iterator(); - if (TransformSpec.TransformType.IDENTITY.equals(transformType)) { - Expression boundPredicate = Expressions.in(partitionField.name(), iterable); - subExp = Expressions.and(subExp, boundPredicate); - } else { - throw new MetaException( - String.format("Partition transforms are not supported via truncate operation: %s", entry.getKey())); - } - } else { - throw new MetaException(String.format("No partition column/transform name by the name: %s", - entry.getKey())); - } - } - finalExp = Expressions.or(finalExp, subExp); - } - } + + Expression predicate = generateExprFromPartitionNames(partNames); DeleteFiles delete = icebergTable.newDelete(); String branchName = context.getProperties().get(Catalogs.SNAPSHOT_REF); if (branchName != null) { delete.toBranch(HiveUtils.getTableSnapshotRef(branchName)); } - delete.deleteFromRowFilter(finalExp); + + delete.deleteFromRowFilter(predicate); delete.commit(); context.putToProperties("truncateSkipDataDeletion", "true"); } - @Override public boolean createHMSTableInHook() { - return createHMSTableInHook; + private Expression generateExprFromPartitionNames(List partNames) throws MetaException { + if (CollectionUtils.isEmpty(partNames)) { + return Expressions.alwaysTrue(); + } + + Map partitionFields = icebergTable.spec().fields().stream() + .collect(Collectors.toMap(PartitionField::name, Function.identity())); + Expression predicate = Expressions.alwaysFalse(); + + for (String partName : partNames) { + try { + Map partitionSpec = Warehouse.makeSpecFromName(partName); + Expression partitionExpr = IcebergTableUtil.generateExprForIdentityPartition( + icebergTable, partitionSpec, partitionFields); + + predicate = Expressions.or(predicate, partitionExpr); + } catch (Exception e) { + throw new MetaException( + "Failed to generate expression for partition: " + partName + ". " + e.getMessage()); + } + } + + return predicate; } private void alterTableProperties(org.apache.hadoop.hive.metastore.api.Table hmsTable, @@ -1029,10 +1016,7 @@ private static UnboundPredicate getPartitionPredicate(PartitionData part String columName = schema.findField(field.sourceId()).name(); TransformSpec transformSpec = TransformSpec.fromString(field.transform().toString(), columName); - UnboundTerm partitionColumn = - ObjectUtils.defaultIfNull(HiveIcebergFilterFactory.toTerm(columName, transformSpec), - Expressions.ref(field.name())); - + UnboundTerm partitionColumn = SchemaUtils.toTerm(transformSpec); return Expressions.equal(partitionColumn, partitionData.get(index, Object.class)); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 2978341b6da5..7554de2c588a 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -866,8 +866,7 @@ public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.m return table.spec().fields().stream() .filter(f -> !f.transform().isVoid()) .map(f -> { - TransformSpec spec = IcebergTableUtil.getTransformSpec( - table, f.transform().toString(), f.sourceId()); + TransformSpec spec = IcebergTableUtil.getTransformSpec(table, f.transform().toString(), f.sourceId()); spec.setFieldName(f.name()); return spec; }) @@ -882,8 +881,7 @@ public Map> getPartitionTransformSpecs( e.getValue().fields().stream() .filter(f -> !f.transform().isVoid()) .map(f -> { - TransformSpec spec = IcebergTableUtil.getTransformSpec( - table, f.transform().toString(), f.sourceId()); + TransformSpec spec = IcebergTableUtil.getTransformSpec(table, f.transform().toString(), f.sourceId()); spec.setFieldName(f.name()); return Pair.of(e.getKey(), spec); })) @@ -893,9 +891,8 @@ public Map> getPartitionTransformSpecs( private List getSortTransformSpec(Table table) { return table.sortOrder().fields().stream().map(s -> - IcebergTableUtil.getTransformSpec(table, s.transform().toString(), s.sourceId()) - ) - .collect(Collectors.toList()); + IcebergTableUtil.getTransformSpec(table, s.transform().toString(), s.sourceId())) + .toList(); } @Override @@ -2024,8 +2021,7 @@ public void validatePartSpec(org.apache.hadoop.hive.ql.metadata.Table hmsTable, * @param hmsTable A Hive table instance. * @param partitionSpec Map containing partition specification given by user. * @return true if we can perform metadata delete, otherwise false. - * @throws SemanticException Exception raised when a partition transform is being used - * or when partition column is not present in the table. + * @throws SemanticException Exception raised when partition column is not present in the table. */ @Override public boolean canUseTruncate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Map partitionSpec) @@ -2037,13 +2033,16 @@ public boolean canUseTruncate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, return false; } - Expression finalExp = IcebergTableUtil.generateExpressionFromPartitionSpec(table, partitionSpec, true); - FindFiles.Builder builder = new FindFiles.Builder(table).withRecordsMatching(finalExp); + Expression partitionExpr = IcebergTableUtil.generateExprForIdentityPartition( + table, partitionSpec, true); + + FindFiles.Builder builder = new FindFiles.Builder(table).withRecordsMatching(partitionExpr); Set dataFiles = Sets.newHashSet(builder.collect()); + boolean result = true; for (DataFile dataFile : dataFiles) { PartitionData partitionData = (PartitionData) dataFile.partition(); - Expression residual = ResidualEvaluator.of(table.spec(), finalExp, false) + Expression residual = ResidualEvaluator.of(table.spec(), partitionExpr, false) .residualFor(partitionData); if (!residual.isEquivalentTo(Expressions.alwaysTrue())) { result = false; @@ -2056,8 +2055,7 @@ public boolean canUseTruncate(org.apache.hadoop.hive.ql.metadata.Table hmsTable, @Override public List getPartitions(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Map partitionSpec, boolean latestSpecOnly) throws SemanticException { - Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - List partNames = IcebergTableUtil.getPartitionNames(table, partitionSpec, latestSpecOnly); + List partNames = IcebergTableUtil.getPartitionNames(conf, hmsTable, partitionSpec, latestSpecOnly); return IcebergTableUtil.convertNameToMetastorePartition(hmsTable, partNames); } @@ -2078,12 +2076,39 @@ public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) @Override public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec, RewritePolicy policy) throws SemanticException { + validatePartSpec(table, partitionSpec, policy); + + boolean isDescTable = SessionStateUtil.getQueryState(conf) + .map(QueryState::getHiveOperation) + .filter(op -> op == HiveOperation.DESCTABLE) + .isPresent(); + + if (!isDescTable) { + return createDummyPartitionHandle(table, partitionSpec); + } + + Partition partition = IcebergTableUtil.getPartition(conf, table, partitionSpec); + + // Populate basic statistics + if (partition != null) { + Map stats = getBasicStatistics(Partish.buildFor(table, partition)); + if (stats != null && !stats.isEmpty()) { + partition.getTPartition().setParameters(stats); + } + } + + return partition; + } + + private static DummyPartition createDummyPartitionHandle( + org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec) + throws SemanticException { try { - String partName = Warehouse.makePartName(partitionSpec, false); - return new DummyPartition(table, partName, partitionSpec); + String partitionName = Warehouse.makePartName(partitionSpec, false); + return new DummyPartition(table, partitionName, partitionSpec); } catch (MetaException e) { - throw new SemanticException("Unable to construct name for dummy partition due to: ", e); + throw new SemanticException("Unable to construct partition name", e); } } @@ -2096,8 +2121,7 @@ public Partition getPartition(org.apache.hadoop.hive.ql.metadata.Table table, */ public List getPartitionNames(org.apache.hadoop.hive.ql.metadata.Table hmsTable, Map partitionSpec) throws SemanticException { - Table icebergTable = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); - return IcebergTableUtil.getPartitionNames(icebergTable, partitionSpec, false); + return IcebergTableUtil.getPartitionNames(conf, hmsTable, partitionSpec, false); } /** diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java index 6f572afd3b85..fbdc36be3a19 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveTableUtil.java @@ -172,8 +172,8 @@ public static void appendFiles(URI fromURI, String format, Table icebergTbl, boo if (isOverwrite) { DeleteFiles delete = transaction.newDelete(); if (partitionSpec != null) { - Expression partitionExpr = - IcebergTableUtil.generateExpressionFromPartitionSpec(icebergTbl, partitionSpec, true); + Expression partitionExpr = IcebergTableUtil.generateExprForIdentityPartition( + icebergTbl, partitionSpec, true); delete.deleteFromRowFilter(partitionExpr); } else { delete.deleteFromRowFilter(Expressions.alwaysTrue()); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java index 6a6b83f6cc15..b85df34405d0 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java @@ -91,6 +91,7 @@ import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.expressions.ResidualEvaluator; +import org.apache.iceberg.expressions.UnboundTerm; import org.apache.iceberg.hive.IcebergCatalogProperties; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.mr.Catalogs; @@ -102,8 +103,8 @@ import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.types.Conversions; -import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.ByteBuffers; import org.apache.iceberg.util.Pair; @@ -260,41 +261,15 @@ static PartitionStatisticsFile getPartitionStatsFile(Table table, long snapshotI * @return iceberg partition spec, always non-null */ public static PartitionSpec spec(Configuration configuration, Schema schema) { - List partitionTransformSpecList = SessionStateUtil + List partitionBy = SessionStateUtil .getResource(configuration, hive_metastoreConstants.PARTITION_TRANSFORM_SPEC) .map(o -> (List) o).orElse(null); - if (partitionTransformSpecList == null) { + if (partitionBy == null) { LOG.warn(PARTITION_TRANSFORM_SPEC_NOT_FOUND); return null; } - PartitionSpec.Builder builder = PartitionSpec.builderFor(schema); - partitionTransformSpecList.forEach(spec -> { - switch (spec.getTransformType()) { - case IDENTITY: - builder.identity(spec.getColumnName().toLowerCase()); - break; - case YEAR: - builder.year(spec.getColumnName()); - break; - case MONTH: - builder.month(spec.getColumnName()); - break; - case DAY: - builder.day(spec.getColumnName()); - break; - case HOUR: - builder.hour(spec.getColumnName()); - break; - case TRUNCATE: - builder.truncate(spec.getColumnName(), spec.getTransformParam()); - break; - case BUCKET: - builder.bucket(spec.getColumnName(), spec.getTransformParam()); - break; - } - }); - return builder.build(); + return SchemaUtils.createPartitionSpec(schema, partitionBy); } public static void updateSpec(Configuration configuration, Table table) { @@ -317,32 +292,8 @@ public static void updateSpec(Configuration configuration, Table table) { LOG.warn(PARTITION_TRANSFORM_SPEC_NOT_FOUND); return; } - partitionTransformSpecList.forEach(spec -> { - switch (spec.getTransformType()) { - case IDENTITY: - updatePartitionSpec.addField(spec.getColumnName()); - break; - case YEAR: - updatePartitionSpec.addField(Expressions.year(spec.getColumnName())); - break; - case MONTH: - updatePartitionSpec.addField(Expressions.month(spec.getColumnName())); - break; - case DAY: - updatePartitionSpec.addField(Expressions.day(spec.getColumnName())); - break; - case HOUR: - updatePartitionSpec.addField(Expressions.hour(spec.getColumnName())); - break; - case TRUNCATE: - updatePartitionSpec.addField(Expressions.truncate(spec.getColumnName(), spec.getTransformParam())); - break; - case BUCKET: - updatePartitionSpec.addField(Expressions.bucket(spec.getColumnName(), spec.getTransformParam())); - break; - } - }); - + partitionTransformSpecList.forEach(spec -> + updatePartitionSpec.addField(SchemaUtils.toTerm(spec))); updatePartitionSpec.commit(); } @@ -489,82 +440,260 @@ public static PartitionData toPartitionData(StructLike key, Types.StructType key public static PartitionData toPartitionData(StructLike sourceKey, Types.StructType sourceKeyType, Types.StructType targetKeyType) { - StructProjection projection = StructProjection.create(sourceKeyType, targetKeyType).wrap(sourceKey); + StructProjection projection = StructProjection.create(sourceKeyType, targetKeyType) + .wrap(sourceKey); return toPartitionData(projection, targetKeyType); } - public static Expression generateExpressionFromPartitionSpec(Table table, Map partitionSpec, + public static Expression generateExprForIdentityPartition(Table table, Map partitionSpec, boolean latestSpecOnly) throws SemanticException { - Map partitionFieldMap = getPartitionFields(table, latestSpecOnly).stream() + + Map partitionFields = getPartitionFields(table, latestSpecOnly).stream() .collect(Collectors.toMap(PartitionField::name, Function.identity())); - Expression finalExp = Expressions.alwaysTrue(); + + return generateExprForIdentityPartition(table, partitionSpec, partitionFields); + } + + public static Expression generateExprForIdentityPartition(Table table, Map partitionSpec, + Map partitionFields) throws SemanticException { + + return buildPartitionExpression( + partitionSpec, + (column, value) -> + buildIdentityPartitionPredicate(table, value, partitionFields.get(column)), + partitionFields::containsKey + ); + } + + public static Expression generateExprFromPartitionSpec(Table table, Map partitionSpec, + boolean latestSpecOnly) throws SemanticException { + + // Group partition fields by source column name to handle partition evolution + // where the same source column may have multiple transforms across different specs + Map> partitionFieldsBySourceColumn = + getPartitionFields(table, latestSpecOnly).stream() + .collect(Collectors.groupingBy( + pf -> table.schema().findColumnName(pf.sourceId())) + ); + + return buildPartitionExpression( + partitionSpec, + (column, value) -> + buildTransformPartitionPredicate(table, value, partitionFieldsBySourceColumn.get(column)), + partitionFieldsBySourceColumn::containsKey + ); + } + + @FunctionalInterface + private interface PartitionPredicateBuilder { + Expression build(String partitionColumn, String partitionValue) throws SemanticException; + } + + private static Expression buildPartitionExpression( + Map partitionSpec, + PartitionPredicateBuilder predicateBuilder, + Predicate fieldValidator) throws SemanticException { + + Expression predicate = Expressions.alwaysTrue(); + for (Map.Entry entry : partitionSpec.entrySet()) { - String partColName = entry.getKey(); - if (partitionFieldMap.containsKey(partColName)) { - PartitionField partitionField = partitionFieldMap.get(partColName); - if (partitionField.transform().isIdentity()) { - final Type partKeyType = table.schema().findField(partitionField.sourceId()).type(); - final Object partKeyVal = Conversions.fromPartitionString(partKeyType, entry.getValue()); - Expression boundPredicate = Expressions.equal(partColName, partKeyVal); - finalExp = Expressions.and(finalExp, boundPredicate); - } else { - throw new SemanticException( - String.format("Partition transforms are not supported here: %s", partColName)); - } - } else { - throw new SemanticException(String.format("No partition column by the name: %s", partColName)); + String partitionColumn = entry.getKey(); + + // Validate field exists + if (!fieldValidator.test(partitionColumn)) { + throw new SemanticException( + "No partition column by the name: %s".formatted(partitionColumn)); } + Expression columnPredicate = predicateBuilder.build(partitionColumn, entry.getValue()); + predicate = Expressions.and(predicate, columnPredicate); + } + + return predicate; + } + + private static Expression buildIdentityPartitionPredicate(Table table, String partitionValue, + PartitionField partitionField) throws SemanticException { + + if (!partitionField.transform().isIdentity()) { + throw new SemanticException( + "Partition transforms are not supported here: %s".formatted(partitionField.name())); + } + Types.NestedField sourceField = table.schema().findField(partitionField.sourceId()); + Object columnValue = Conversions.fromPartitionString(sourceField.type(), partitionValue); + + return Expressions.equal(partitionField.name(), columnValue); + } + + private static Expression buildTransformPartitionPredicate(Table table, String partitionValue, + List partitionFields) { + + // Get source field type from first partition field (all share same source) + Types.NestedField sourceField = table.schema().findField( + partitionFields.getFirst().sourceId()); + Object columnValue = Conversions.fromPartitionString(sourceField.type(), partitionValue); + + Expression predicate = Expressions.alwaysFalse(); + + // Create OR expression for each transform on this source column + for (PartitionField partitionField : partitionFields) { + // Apply the transform to the source value + @SuppressWarnings("unchecked") + Transform transform = (Transform) partitionField.transform(); + Object transformedValue = transform.bind(sourceField.type()).apply(columnValue); + + TransformSpec transformSpec = TransformSpec.fromString(transform.toString().toUpperCase(), sourceField.name()); + UnboundTerm term = SchemaUtils.toTerm(transformSpec); + + predicate = Expressions.or( + predicate, Expressions.equal(term, transformedValue)); } - return finalExp; + + return predicate; } public static List getPartitionFields(Table table, boolean latestSpecOnly) { - return latestSpecOnly ? table.spec().fields() : - table.specs().values().stream() - .flatMap(spec -> spec.fields().stream() - .filter(f -> !f.transform().isVoid())) - .distinct() - .collect(Collectors.toList()); + if (latestSpecOnly) { + return table.spec().fields(); + } + return table.specs().values().stream() + .flatMap(spec -> spec.fields().stream()) + .filter(f -> !f.transform().isVoid()) + .toList(); + } + + /** + * Returns a partition matching the given partition spec. + * With partition evolution, multiple partitions may match; returns the one from the highest spec ID. + * @param conf Configuration + * @param table Hive table + * @param partitionSpec Partition specification with source column names and values + * @return Partition matching the spec, or null if no match found + */ + public static Partition getPartition(Configuration conf, + org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec) + throws SemanticException { + // Get partitions sorted by spec ID descending + List partitionNames = getPartitionNames(conf, table, partitionSpec, false, + Map.Entry.comparingByValue(Comparator.reverseOrder())); + + if (partitionNames.isEmpty()) { + return null; + } + + // Find first partition with matching spec size (highest spec ID due to sort order) + Optional partitionName = partitionNames.stream() + .filter(p -> hasMatchingSpecSize(p, partitionSpec.size())) + .findFirst(); + + return partitionName + .map(p -> new DummyPartition(table, p, partitionSpec)) + .orElse(null); + } + + /** + * Checks if a partition name has the expected number of fields. + */ + private static boolean hasMatchingSpecSize( + String partitionName, int expectedSpecSize) { + try { + return Warehouse.makeSpecFromName(partitionName).size() == expectedSpecSize; + } catch (MetaException e) { + return false; + } } /** - * Returns a list of partition names satisfying the provided partition spec. - * @param table Iceberg table - * @param partSpecMap Partition Spec used as the criteria for filtering - * @param latestSpecOnly when True, returns partitions with the current spec only, else - any specs - * @return List of partition names + * Returns partition names matching the provided partition spec. + * @param conf Configuration + * @param table Hive table + * @param partSpecMap Partition spec for filtering + * @param latestSpecOnly if true, return only partitions from latest spec; otherwise all specs + * @return List of partition names sorted by natural order */ - public static List getPartitionNames(Table table, Map partSpecMap, + public static List getPartitionNames(Configuration conf, + org.apache.hadoop.hive.ql.metadata.Table table, Map partSpecMap, boolean latestSpecOnly) throws SemanticException { - Expression expression = IcebergTableUtil.generateExpressionFromPartitionSpec( - table, partSpecMap, latestSpecOnly); + return getPartitionNames(conf, table, partSpecMap, latestSpecOnly, Map.Entry.comparingByKey()); + } + + /** + * Returns partition names matching the provided partition spec, sorted by the given comparator. + * + * @param specIdComparator Comparator for Entry<partitionPath, specId> + */ + private static List getPartitionNames(Configuration conf, + org.apache.hadoop.hive.ql.metadata.Table table, Map partitionSpec, boolean latestSpecOnly, + Comparator> specIdComparator) throws SemanticException { + Table icebergTable = getTable(conf, table.getTTable()); + + Expression partitionExpr = IcebergTableUtil.generateExprFromPartitionSpec( + icebergTable, partitionSpec, latestSpecOnly); + + int latestSpecId = icebergTable.spec().specId(); + Types.StructType partitionType = Partitioning.partitionType(icebergTable); + PartitionsTable partitionsTable = (PartitionsTable) MetadataTableUtils.createMetadataTableInstance( - table, MetadataTableType.PARTITIONS); + icebergTable, MetadataTableType.PARTITIONS); try (CloseableIterable fileScanTasks = partitionsTable.newScan().planFiles()) { return FluentIterable.from(fileScanTasks) .transformAndConcat(task -> task.asDataTask().rows()) - .transform(row -> { - StructLike data = row.get(IcebergTableUtil.PART_IDX, StructProjection.class); - PartitionSpec spec = table.specs().get(row.get(IcebergTableUtil.SPEC_IDX, Integer.class)); - return Maps.immutableEntry( - IcebergTableUtil.toPartitionData( - data, Partitioning.partitionType(table), spec.partitionType()), - spec); - }).filter(e -> { - ResidualEvaluator resEval = ResidualEvaluator.of(e.getValue(), - expression, false); - return e.getValue().isPartitioned() && - resEval.residualFor(e.getKey()).isEquivalentTo(Expressions.alwaysTrue()) && - (e.getValue().specId() == table.spec().specId() || !latestSpecOnly); - - }).transform(e -> e.getValue().partitionToPath(e.getKey())).toSortedList( - Comparator.naturalOrder()); + .transform(row -> extractPartitionDataAndSpec(row, icebergTable, partitionType)) + .filter(entry -> matchesPartition(entry, partitionExpr, latestSpecOnly, latestSpecId)) + // Create (partitionPath, specId) entries for sorting + .transform(entry -> Maps.immutableEntry( + entry.getValue().partitionToPath(entry.getKey()), + entry.getValue().specId())) + .toSortedList(specIdComparator).stream() + .map(Map.Entry::getKey) + .toList(); } catch (IOException e) { - throw new SemanticException( - String.format("Error while fetching the partitions due to: %s", e)); + throw new SemanticException("Error while fetching the partitions", e); + } + } + + /** + * Checks if a partition matches the filter expression and spec requirements. + */ + private static boolean matchesPartition(Map.Entry entry, + Expression filterExpression, boolean latestSpecOnly, int latestSpecId) { + PartitionData partitionData = entry.getKey(); + PartitionSpec spec = entry.getValue(); + + // Filter unpartitioned tables + if (!spec.isPartitioned()) { + return false; + } + // Filter by spec ID if requested + if (latestSpecOnly && spec.specId() != latestSpecId) { + return false; } + // Check if partition matches filter expression + ResidualEvaluator evaluator = + ResidualEvaluator.of(spec, filterExpression, false); + + return evaluator + .residualFor(partitionData) + .isEquivalentTo(Expressions.alwaysTrue()); + } + + /** + * Extracts partition data and spec from a partitions metadata table row. + */ + private static Map.Entry extractPartitionDataAndSpec( + StructLike row, Table icebergTable, Types.StructType partitionType) { + + StructLike rawPartition = + row.get(IcebergTableUtil.PART_IDX, StructProjection.class); + + PartitionSpec spec = icebergTable.specs().get( + row.get(IcebergTableUtil.SPEC_IDX, Integer.class)); + + return Maps.immutableEntry( + IcebergTableUtil.toPartitionData( + rawPartition, partitionType, spec.partitionType()), + spec); } public static PartitionSpec getPartitionSpec(Table icebergTable, String partitionPath) @@ -596,7 +725,7 @@ public static TransformSpec getTransformSpec(Table table, String transformName, public static List readColStats(Table table, Long snapshotId, Predicate filter) { List colStats = Lists.newArrayList(); - String statsPath = IcebergTableUtil.getColStatsPath(table, snapshotId); + String statsPath = IcebergTableUtil.getColStatsPath(table, snapshotId); if (statsPath == null) { return colStats; } @@ -643,8 +772,7 @@ public static > Set getPartitionNames(Table ice Set partitions = Sets.newHashSet(); int tableSpecId = icebergTable.spec().specId(); for (T file : files) { - if (latestSpecOnly == null || Boolean.TRUE.equals(latestSpecOnly) && file.specId() == tableSpecId || - Boolean.FALSE.equals(latestSpecOnly) && file.specId() != tableSpecId) { + if (latestSpecOnly == null || latestSpecOnly.equals(file.specId() == tableSpecId)) { String partName = icebergTable.specs().get(file.specId()).partitionToPath(file.partition()); partitions.add(partName); } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java new file mode 100644 index 000000000000..8444da80510f --- /dev/null +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/SchemaUtils.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.mr.hive; + +import java.util.List; +import org.apache.hadoop.hive.ql.parse.TransformSpec; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.UnboundTerm; + +public class SchemaUtils { + + private static final String UNSUPPORTED_TRANSFORM = "Unsupported transform: %s"; + + private SchemaUtils() { + } + + public static UnboundTerm toTerm(TransformSpec spec) { + if (spec == null) { + return null; + } + + return switch (spec.getTransformType()) { + case YEAR -> + Expressions.year(spec.getColumnName()); + case MONTH -> + Expressions.month(spec.getColumnName()); + case DAY -> + Expressions.day(spec.getColumnName()); + case HOUR -> + Expressions.hour(spec.getColumnName()); + case TRUNCATE -> + Expressions.truncate(spec.getColumnName(), spec.getTransformParam()); + case BUCKET -> + Expressions.bucket(spec.getColumnName(), spec.getTransformParam()); + case IDENTITY -> + Expressions.ref(spec.getColumnName()); + default -> + throw new UnsupportedOperationException( + UNSUPPORTED_TRANSFORM.formatted(spec.getTransformType())); + }; + } + + public static PartitionSpec createPartitionSpec(Schema schema, List partitionBy) { + if (partitionBy.isEmpty()) { + return PartitionSpec.unpartitioned(); + } + + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + + partitionBy.forEach(spec -> { + switch (spec.getTransformType()) { + case IDENTITY -> + specBuilder.identity(spec.getColumnName().toLowerCase()); + case YEAR -> + specBuilder.year(spec.getColumnName()); + case MONTH -> + specBuilder.month(spec.getColumnName()); + case DAY -> + specBuilder.day(spec.getColumnName()); + case HOUR -> + specBuilder.hour(spec.getColumnName()); + case TRUNCATE -> + specBuilder.truncate(spec.getColumnName(), spec.getTransformParam()); + case BUCKET -> + specBuilder.bucket(spec.getColumnName(), spec.getTransformParam()); + default -> + throw new UnsupportedOperationException( + UNSUPPORTED_TRANSFORM.formatted(spec.getTransformType())); + } + }); + + return specBuilder.build(); + } +} diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java index b0360112e7af..c2580c113983 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergPartitions.java @@ -256,4 +256,25 @@ public void testTruncateTransform() throws IOException { HiveIcebergTestUtils.validateDataWithSQL(shell, "part_test", records, "id"); } + + @Test + public void testShowPartitionsWithTransform() { + Schema schema = new Schema( + optional(1, "id", Types.IntegerType.get()), + optional(2, "part_field", Types.StringType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("part_field", 2).build(); + List records = TestHelper.RecordsBuilder.newInstance(schema) + .add(1L, "Part1") + .add(2L, "Part2") + .add(3L, "Art3") + .build(); + testTables.createTable(shell, "part_test", schema, spec, fileFormat, records); + + List rows = shell.executeStatement("SHOW PARTITIONS part_test"); + Assert.assertEquals(2, rows.size()); + + rows = shell.executeStatement("SHOW PARTITIONS part_test PARTITION(part_field='Art3')"); + Assert.assertEquals(1, rows.size()); + Assert.assertEquals("part_field_trunc=Ar", rows.get(0)[0]); + } } diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java index 3d12cfb1179a..ca5801aea165 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java @@ -51,6 +51,7 @@ import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.BaseTable; import org.apache.iceberg.FileFormat; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; @@ -959,8 +960,8 @@ public void testCreatePartitionedTableWithColumnComments() { List rows = shell.executeStatement("DESCRIBE default.partitioned_with_comment_table"); List columns = icebergTable.schema().columns(); - // The partition transform information is 3 extra lines, and 2 more line for the columns - Assert.assertEquals(columns.size() + 5, rows.size()); + // The partition transform information and partition information is 6 extra lines, and 4 more line for the columns + Assert.assertEquals(columns.size() + 10, rows.size()); for (int i = 0; i < columns.size(); i++) { Types.NestedField field = columns.get(i); Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(), @@ -1323,8 +1324,8 @@ public void testAlterTableRenamePartitionColumn() throws Exception { shell.executeStatement("ALTER TABLE default.customers SET PARTITION SPEC (region, city)"); List result = shell.executeStatement("DESCRIBE default.customers"); - Assert.assertArrayEquals(new String[] {"region", "IDENTITY", null}, result.get(8)); - Assert.assertArrayEquals(new String[] {"city", "IDENTITY", null}, result.get(9)); + Assert.assertArrayEquals(new String[] {"region", "IDENTITY", null}, result.get(14)); + Assert.assertArrayEquals(new String[] {"city", "IDENTITY", null}, result.get(15)); } @Test @@ -1491,21 +1492,25 @@ public void testMetaHookWithUndefinedAlterOperationType() throws Exception { } @Test - public void testCommandsWithPartitionClauseThrow() { + public void testCommandsWithPartitionClause() throws IOException { TableIdentifier target = TableIdentifier.of("default", "target"); PartitionSpec spec = PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA) .identity("last_name").build(); testTables.createTable(shell, target.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, spec, FileFormat.PARQUET, ImmutableList.of()); + PartitionData partitionData = new PartitionData(spec.partitionType()); + partitionData.set(0, "Johnson"); + org.apache.iceberg.Table icebergTable = testTables.loadTable(target); + testTables.appendIcebergTable(shell.getHiveConf(), icebergTable, FileFormat.PARQUET, partitionData, + HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS); String[] commands = { "DESCRIBE target PARTITION (last_name='Johnson')" }; for (String command : commands) { - Assertions.assertThatThrownBy(() -> shell.executeStatement(command)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Using partition spec in query is unsupported"); + Assertions.assertThatCode(() -> shell.executeStatement(command)) + .doesNotThrowAnyException(); } } diff --git a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q new file mode 100644 index 000000000000..797a91214e1c --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_partial_part_spec.q @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS ice_t; + +CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG; + +ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +); + +INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6"); + +-- this spec will resolve "hello5" to "he" due to truncate(2, b) which don't correspond to any partition available of size 2 +DESC FORMATTED ice_t +PARTITION (c = 6, b = "hello5"); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q new file mode 100644 index 000000000000..d58e26f20d24 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/negative/desc_ice_tbl_wrong_part_spec.q @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS ice_t; + +CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG; + +ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +); + +INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6"); + +-- this spec will resolve "bello5" to "be" due to truncate(2, b) which don't correspond to any partition available of size 3 +DESC FORMATTED ice_t +PARTITION (c = 6, d = "hello6", b = "bello5"); \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q new file mode 100644 index 000000000000..49948d59e5ab --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/desc_ice_tbl_part_spec.q @@ -0,0 +1,64 @@ +DROP TABLE IF EXISTS ice_t; + +CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG; + +INSERT INTO TABLE ice_t +VALUES (1, "hello1", 2, "hello2"); + +INSERT INTO TABLE ice_t +VALUES (3, "hello3", 4, "hello4"); + +INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6"); + +DESC EXTENDED ice_t +PARTITION (c = 6, d = "hello6"); + +DESC FORMATTED ice_t +PARTITION (c = 6, d = "hello6"); + +ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +); + +INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8"); + +ALTER TABLE ice_t +SET PARTITION SPEC ( + bucket(16, c), + d, + truncate(2, b) +); + +INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8"); + +EXPLAIN DESC EXTENDED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7"); + +EXPLAIN DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7"); + +DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7"); + +-- this will also generate a valid result as "hello10" will resolve to "he" due to truncate(2, b) +DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello10"); + +DESC FORMATTED ice_t +PARTITION (c = 4, d = "hello4"); diff --git a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out new file mode 100644 index 000000000000..7061bd176db6 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_partial_part_spec.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: DROP TABLE IF EXISTS ice_t +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: DROP TABLE IF EXISTS ice_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +FAILED: SemanticException [Error 10006]: Partition not found {c=6, b=hello5} diff --git a/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out new file mode 100644 index 000000000000..206d80c2244c --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/negative/desc_ice_tbl_wrong_part_spec.q.out @@ -0,0 +1,62 @@ +PREHOOK: query: DROP TABLE IF EXISTS ice_t +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: DROP TABLE IF EXISTS ice_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +FAILED: SemanticException [Error 10006]: Partition not found {c=6, d=hello6, b=bello5} diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out index 118552ac4d5d..910e48e4214e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out @@ -180,6 +180,11 @@ a int b string c string +# Partition Information +# col_name data_type comment +b string Transform: identity +c string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -451,6 +456,11 @@ a int b string c string +# Partition Information +# col_name data_type comment +b string Transform: identity +c string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -722,6 +732,11 @@ a int b string c string +# Partition Information +# col_name data_type comment +b string Transform: identity +c string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1055,6 +1070,12 @@ b double c int d string +# Partition Information +# col_name data_type comment +b double Transform: identity +c int Transform: identity +d string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1496,6 +1517,12 @@ b double c int d string +# Partition Information +# col_name data_type comment +b double Transform: identity +c int Transform: identity +d string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1937,6 +1964,12 @@ b double c int d string +# Partition Information +# col_name data_type comment +b double Transform: identity +c int Transform: identity +d string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out index 2e55bbd42b7c..55bfee6eb031 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out @@ -139,6 +139,10 @@ POSTHOOK: Input: default@tbl_orc a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -413,6 +417,10 @@ POSTHOOK: Input: default@tbl_parquet a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -764,6 +772,10 @@ POSTHOOK: Input: default@tbl_parquet_int a int b int +# Partition Information +# col_name data_type comment +b int Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1115,6 +1127,10 @@ POSTHOOK: Input: default@tbl_parquet_double a int b double +# Partition Information +# col_name data_type comment +b double Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -1412,6 +1428,10 @@ POSTHOOK: Input: default@tbl_avro a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out index c0a81219df6d..0d1700ff07a9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/ctas_iceberg_partitioned_orc.q.out @@ -277,6 +277,11 @@ a int b string c int +# Partition Information +# col_name data_type comment +a int Transform: bucket[16] +b string Transform: truncate[3] + # Partition Transform Information # col_name transform_type a BUCKET[16] diff --git a/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out new file mode 100644 index 000000000000..9ae5ea93e5f8 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/desc_ice_tbl_part_spec.q.out @@ -0,0 +1,336 @@ +PREHOOK: query: DROP TABLE IF EXISTS ice_t +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: DROP TABLE IF EXISTS ice_t +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_t +POSTHOOK: query: CREATE EXTERNAL TABLE ice_t ( + a INT, + b STRING +) +PARTITIONED BY ( + c INT, + d STRING +) +WRITE LOCALLY +ORDERED BY a DESC +STORED BY ICEBERG +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (1, "hello1", 2, "hello2") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (1, "hello1", 2, "hello2") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (3, "hello3", 4, "hello4") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (3, "hello3", 4, "hello4") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (5, "hello5", 6, "hello6") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: DESC EXTENDED ice_t +PARTITION (c = 6, d = "hello6") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: DESC EXTENDED ice_t +PARTITION (c = 6, d = "hello6") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY + +#### A masked pattern was here #### +PREHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 6, d = "hello6") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 6, d = "hello6") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +c int Transform: identity +d string Transform: identity + +# Partition Transform Information +# col_name transform_type +c IDENTITY +d IDENTITY + +# Detailed Partition Information +Partition Value: [6, hello6] +Database: default +Table: ice_t +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1 + totalSize #Masked# + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] +PREHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + c, + d, + truncate(2, b) +) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + bucket(16, c), + d, + truncate(2, b) +) +PREHOOK: type: ALTERTABLE_SETPARTSPEC +PREHOOK: Input: default@ice_t +POSTHOOK: query: ALTER TABLE ice_t +SET PARTITION SPEC ( + bucket(16, c), + d, + truncate(2, b) +) +POSTHOOK: type: ALTERTABLE_SETPARTSPEC +POSTHOOK: Input: default@ice_t +POSTHOOK: Output: default@ice_t +PREHOOK: query: INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8") +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_t +POSTHOOK: query: INSERT INTO TABLE ice_t +VALUES (7, "hello7", 8, "hello8") +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_t +PREHOOK: query: EXPLAIN DESC EXTENDED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: EXPLAIN DESC EXTENDED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +Stage-1 + Fetch Operator + limit:-1 + Stage-0 + Describe Table{"table:":"default.ice_t","partition:":{"b":"hello7","c":"8","d":"hello8"},"extended:":"true"} + +PREHOOK: query: EXPLAIN DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: EXPLAIN DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +Stage-1 + Fetch Operator + limit:-1 + Stage-0 + Describe Table{"table:":"default.ice_t","partition:":{"b":"hello7","c":"8","d":"hello8"},"formatted:":"true"} + +PREHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello7") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +d string Transform: identity +b string Transform: truncate[2] +c int Transform: bucket[16] + +# Partition Transform Information +# col_name transform_type +d IDENTITY +b TRUNCATE[2] +c BUCKET[16] + +# Detailed Partition Information +Partition Value: [hello8, hello7, 8] +Database: default +Table: ice_t +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1 + totalSize #Masked# + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] +PREHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello10") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 8, d = "hello8", b = "hello10") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +d string Transform: identity +b string Transform: truncate[2] +c int Transform: bucket[16] + +# Partition Transform Information +# col_name transform_type +d IDENTITY +b TRUNCATE[2] +c BUCKET[16] + +# Detailed Partition Information +Partition Value: [hello8, hello10, 8] +Database: default +Table: ice_t +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1 + totalSize #Masked# + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] +PREHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 4, d = "hello4") +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_t +POSTHOOK: query: DESC FORMATTED ice_t +PARTITION (c = 4, d = "hello4") +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_t +# col_name data_type comment +a int +b string +c int +d string + +# Partition Information +# col_name data_type comment +d string Transform: identity +b string Transform: truncate[2] +c int Transform: bucket[16] + +# Partition Transform Information +# col_name transform_type +d IDENTITY +b TRUNCATE[2] +c BUCKET[16] + +# Detailed Partition Information +Partition Value: [hello4, null, 4] +Database: default +Table: ice_t +#### A masked pattern was here #### +Partition Parameters: + numFiles 1 + numRows 1 + totalSize #Masked# + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [FieldSchema(name:a, type:int, comment:Transform: identity, Sort direction: DESC, Null sort order: NULLS_LAST)] diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out index 09ef4d1ac96c..fb1cdbcaf12a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out @@ -117,6 +117,16 @@ truncate_field string bucket_field int identity_field int +# Partition Information +# col_name data_type comment +year_field date Transform: year +month_field date Transform: month +day_field date Transform: day +hour_field timestamp Transform: hour +truncate_field string Transform: truncate[2] +bucket_field int Transform: bucket[2] +identity_field int Transform: identity + # Partition Transform Information # col_name transform_type year_field YEAR @@ -180,6 +190,16 @@ truncate_field string bucket_field int identity_field int +# Partition Information +# col_name data_type comment +year_field date Transform: year +month_field date Transform: month +day_field date Transform: day +hour_field timestamp Transform: hour +truncate_field string Transform: truncate[2] +bucket_field int Transform: bucket[2] +identity_field int Transform: identity + # Partition Transform Information # col_name transform_type year_field YEAR @@ -237,6 +257,10 @@ POSTHOOK: Input: default@ice_t_identity_part a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out index c7e7c4e4acf7..74d891eefc1e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_drop_column.q.out @@ -30,6 +30,11 @@ intcol int pcol string datecol date +# Partition Information +# col_name data_type comment +pcol string Transform: identity +datecol date Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out index 000b8b0ba819..c68cb256cd60 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition.q.out @@ -481,6 +481,10 @@ strcol string intcol int pcol int +# Partition Information +# col_name data_type comment +pcol int Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -1675,6 +1679,11 @@ age int country string state string +# Partition Information +# col_name data_type comment +country string Transform: identity +state string Transform: identity + # Partition Transform Information # col_name transform_type country IDENTITY @@ -2727,6 +2736,11 @@ age int country string state string +# Partition Information +# col_name data_type comment +country string Transform: identity +state string Transform: identity + # Partition Transform Information # col_name transform_type country IDENTITY @@ -3418,6 +3432,10 @@ datecol date intcol int pcol bigint +# Partition Information +# col_name data_type comment +pcol bigint Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -4108,6 +4126,10 @@ datecol date intcol int pcol double +# Partition Information +# col_name data_type comment +pcol double Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -4798,6 +4820,10 @@ datecol date intcol int pcol decimal(10,6) +# Partition Information +# col_name data_type comment +pcol decimal(10,6) Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out index 1cbaf6d004bf..fcd1c17b24ed 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out @@ -612,6 +612,10 @@ bigintcol bigint intcol int pcol date +# Partition Information +# col_name data_type comment +pcol date Transform: year + # Partition Transform Information # col_name transform_type pcol YEAR @@ -1304,6 +1308,10 @@ bigintcol bigint pcol date intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: month + # Partition Transform Information # col_name transform_type pcol MONTH @@ -1996,6 +2004,10 @@ pcol date bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: day + # Partition Transform Information # col_name transform_type pcol DAY @@ -2447,6 +2459,10 @@ pcol string bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol string Transform: truncate[2] + # Partition Transform Information # col_name transform_type pcol TRUNCATE[2] @@ -2882,6 +2898,10 @@ pcol string bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol string Transform: bucket[16] + # Partition Transform Information # col_name transform_type pcol BUCKET[16] @@ -3105,6 +3125,10 @@ POSTHOOK: Input: default@ice_parquet_decimal_transform_bucket # col_name data_type comment pcol decimal(38,0) +# Partition Information +# col_name data_type comment +pcol decimal(38,0) Transform: bucket[16] + # Partition Transform Information # col_name transform_type pcol BUCKET[16] diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out index 0a724298b074..de49a0d3b366 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_with_evolution.q.out @@ -164,6 +164,10 @@ POSTHOOK: Input: default@testice1000 a int b string +# Partition Information +# col_name data_type comment +b string Transform: truncate[2] + # Partition Transform Information # col_name transform_type b TRUNCATE[2] diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out index 7aaec8f1924e..063b6389863e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition.q.out @@ -263,6 +263,10 @@ strcol string intcol int pcol int +# Partition Information +# col_name data_type comment +pcol int Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -1231,6 +1235,11 @@ age int country string state string +# Partition Information +# col_name data_type comment +country string Transform: identity +state string Transform: identity + # Partition Transform Information # col_name transform_type country IDENTITY @@ -1708,6 +1717,10 @@ bigintcol bigint intcol int pcol date +# Partition Information +# col_name data_type comment +pcol date Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -2172,6 +2185,10 @@ datecol date intcol int pcol bigint +# Partition Information +# col_name data_type comment +pcol bigint Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -2636,6 +2653,10 @@ datecol date intcol int pcol double +# Partition Information +# col_name data_type comment +pcol double Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY @@ -3100,6 +3121,10 @@ datecol date intcol int pcol decimal(10,6) +# Partition Information +# col_name data_type comment +pcol decimal(10,6) Transform: identity + # Partition Transform Information # col_name transform_type pcol IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out index 7b98a5349057..12745b17f098 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_overwrite_partition_transforms.q.out @@ -608,6 +608,10 @@ bigintcol bigint intcol int pcol date +# Partition Information +# col_name data_type comment +pcol date Transform: year + # Partition Transform Information # col_name transform_type pcol YEAR @@ -1274,6 +1278,10 @@ bigintcol bigint pcol date intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: month + # Partition Transform Information # col_name transform_type pcol MONTH @@ -1944,6 +1952,10 @@ pcol date bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol date Transform: day + # Partition Transform Information # col_name transform_type pcol DAY @@ -2395,6 +2407,10 @@ pcol string bigintcol bigint intcol int +# Partition Information +# col_name data_type comment +pcol string Transform: truncate[2] + # Partition Transform Information # col_name transform_type pcol TRUNCATE[2] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out index c6a7243ec961..5036ca420f88 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/hadoop_catalog_create_table.q.out @@ -92,6 +92,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -170,6 +175,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -256,6 +266,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -340,6 +355,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -410,6 +430,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out index b0b02dda8a84..2915bf7d774d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution.q.out @@ -273,6 +273,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY @@ -865,6 +870,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out index d0195d0d475e..3af63c58b6c2 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution2.q.out @@ -140,6 +140,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -223,6 +227,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out index 4b5738e6b7a9..924c6d9953fa 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_ordered.q.out @@ -134,6 +134,10 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -280,6 +284,10 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out index 1f6be0ebf87a..61a931d74a33 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_dyn_spec_w_filter.q.out @@ -228,6 +228,11 @@ event_id int event_time timestamp with local time zone event_src string +# Partition Information +# col_name data_type comment +event_src string Transform: truncate[3] +event_time timestamp with local time zone Transform: month + # Partition Transform Information # col_name transform_type event_src TRUNCATE[3] @@ -334,6 +339,11 @@ event_id int event_time timestamp with local time zone event_src string +# Partition Information +# col_name data_type comment +event_src string Transform: truncate[3] +event_time timestamp with local time zone Transform: month + # Partition Transform Information # col_name transform_type event_src TRUNCATE[3] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out index 96f9e6fe34c1..bc786c1e8dc8 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out @@ -180,6 +180,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY @@ -288,6 +293,11 @@ dept_id bigint team_id bigint company_id bigint +# Partition Information +# col_name data_type comment +company_id bigint Transform: identity +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type company_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out index ef881a53ba39..5508bdca124c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partitioned.q.out @@ -175,6 +175,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -274,6 +278,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -508,6 +516,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -611,6 +623,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out index 9e59529a9e2f..440f6334f114 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out @@ -211,6 +211,10 @@ last_name string dept_id bigint address string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -311,6 +315,10 @@ last_name string dept_id bigint address string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out index ba16b014e3c6..4e120fb8c50d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition.q.out @@ -181,6 +181,12 @@ dept_id bigint city string registration_date date +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -287,6 +293,12 @@ dept_id bigint city string registration_date date +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -400,6 +412,12 @@ dept_id bigint city string registration_date date +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out index 23ff2c3bafa8..1d1143f4b635 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution.q.out @@ -162,6 +162,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -273,6 +279,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -384,6 +396,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -494,6 +512,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -604,6 +628,12 @@ registration_date date dept_id bigint city string +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity +city string Transform: identity +registration_date date Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out index f3a5e2964b5a..b01185bb6911 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_single_partition_with_evolution2.q.out @@ -109,6 +109,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY @@ -199,6 +203,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY @@ -303,6 +311,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY @@ -407,6 +419,10 @@ POSTHOOK: Input: default@ice_orc a string b bigint +# Partition Information +# col_name data_type comment +a string Transform: identity + # Partition Transform Information # col_name transform_type a IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out index 2a1860b2b3d5..fa5dcd05e101 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_bucket.q.out @@ -68,6 +68,10 @@ id string key int value string +# Partition Information +# col_name data_type comment +key int Transform: bucket[8] + # Partition Transform Information # col_name transform_type key BUCKET[8] @@ -182,6 +186,10 @@ id string key int value string +# Partition Information +# col_name data_type comment +key int Transform: bucket[8] + # Partition Transform Information # col_name transform_type key BUCKET[8] diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out index 52ef77680462..62070278505d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_minor_compaction_partition_evolution.q.out @@ -96,6 +96,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -175,6 +179,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY @@ -280,6 +288,10 @@ first_name string last_name string dept_id bigint +# Partition Information +# col_name data_type comment +dept_id bigint Transform: identity + # Partition Transform Information # col_name transform_type dept_id IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out index e31fb1f787e2..5f31e752db3f 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out @@ -881,6 +881,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out index 1902755ec3a1..f1a017215639 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out @@ -610,6 +610,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out index bbd6c589e90a..2feda580b67a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out @@ -513,6 +513,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out index 0bb8f5837f73..7ea7605467ca 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc.q.out @@ -50,6 +50,10 @@ POSTHOOK: Input: default@mat1 c int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -138,6 +142,10 @@ POSTHOOK: Input: default@mat2 c int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out index d08fb3ec1095..770cc967d0bc 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/mv_iceberg_partitioned_orc2.q.out @@ -50,6 +50,11 @@ POSTHOOK: Input: default@mat1 b string c int +# Partition Information +# col_name data_type comment +b string Transform: bucket[16] +c int Transform: truncate[3] + # Partition Transform Information # col_name transform_type b BUCKET[16] @@ -139,6 +144,11 @@ POSTHOOK: Input: default@mat2 b string c int +# Partition Information +# col_name data_type comment +b string Transform: bucket[16] +c int Transform: truncate[3] + # Partition Transform Information # col_name transform_type b BUCKET[16] diff --git a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out index 1c4d2e05cdb8..a44d9394025b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/row_count.q.out @@ -85,6 +85,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY @@ -172,6 +177,11 @@ tradets timestamp p1 string p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out b/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out index c3c309b656aa..24fff78df880 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/show_partitions_test.q.out @@ -37,6 +37,11 @@ equality_delete_record_count bigint Count of records in equality d equality_delete_file_count int Count of equality delete files last_updated_at timestamp with local time zone Commit time of snapshot that last updated this partition last_updated_snapshot_id bigint Id of snapshot that last updated this partition + +# Partition Information +# col_name data_type comment +d_part int Transform: identity +e_part int Transform: identity PREHOOK: query: select * from default.ice1.partitions PREHOOK: type: QUERY PREHOOK: Input: default@ice1 diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out index 0d6f6a02a622..be765c27120a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out @@ -81,6 +81,10 @@ POSTHOOK: Input: default@test_truncate a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY @@ -185,6 +189,10 @@ POSTHOOK: Input: default@test_truncate a int b string +# Partition Information +# col_name data_type comment +b string Transform: identity + # Partition Transform Information # col_name transform_type b IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out index f37b6a27174a..cd0ce562a725 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out @@ -762,6 +762,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out index 95a19640d864..fdf2679f5b2a 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out @@ -531,6 +531,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out index 4f3d0dd65dda..acc7794e12ce 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out @@ -435,6 +435,11 @@ b string a int p2 string +# Partition Information +# col_name data_type comment +p1 string Transform: identity +p2 string Transform: identity + # Partition Transform Information # col_name transform_type p1 IDENTITY diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java index db1e6a6abbac..c5315f83bc04 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableAnalyzer.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.ddl.DDLWork; -import org.apache.hadoop.hive.ql.ddl.table.partition.PartitionUtils; import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType; import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.exec.Task; @@ -32,7 +31,6 @@ import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -75,15 +73,11 @@ public void analyzeInternal(ASTNode root) throws SemanticException { } Table table = getTable(tableName); - // process the second child, if exists, node to get partition spec(s) - Map partitionSpec = getPartitionSpec(db, tableTypeExpr, tableName); - if (partitionSpec != null) { - // validate that partition exists - PartitionUtils.getPartition(db, table, partitionSpec, true); - } + // process the second child, if exists, node to get partition + Partition partition = getPartition(db, tableTypeExpr, table); // process the third child node,if exists, to get partition spec(s) - String columnPath = getColumnPath(tableTypeExpr, tableName, partitionSpec); + String columnPath = getColumnPath(tableTypeExpr, tableName, partition); boolean showColStats = false; boolean isFormatted = false; @@ -102,7 +96,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { inputs.add(new ReadEntity(table)); - DescTableDesc desc = new DescTableDesc(ctx.getResFile(), tableName, partitionSpec, columnPath, isExt, isFormatted); + DescTableDesc desc = new DescTableDesc(ctx.getResFile(), tableName, partition, columnPath, isExt, isFormatted); Task task = TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)); rootTasks.add(task); @@ -117,7 +111,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { * Example: lintString.$elem$.myint. * Return table name for column name if no column has been specified. */ - private String getColumnPath(ASTNode node, TableName tableName, Map partitionSpec) { + private String getColumnPath(ASTNode node, TableName tableName, Partition partition) { // if this ast has only one child, then no column name specified. if (node.getChildCount() == 1) { return null; @@ -125,7 +119,8 @@ private String getColumnPath(ASTNode node, TableName tableName, Map 1) { - ASTNode columnNode = (partitionSpec == null) ? (ASTNode) node.getChild(1) : (ASTNode) node.getChild(2); + ASTNode columnNode = (partition == null) ? + (ASTNode) node.getChild(1) : (ASTNode) node.getChild(2); if (columnNode != null) { return String.join(".", tableName.getNotEmptyDbTable(), DDLUtils.getFQName(columnNode)); } @@ -134,7 +129,8 @@ private String getColumnPath(ASTNode node, TableName tableName, Map getPartitionSpec(Hive db, ASTNode node, TableName tableName) throws SemanticException { + private Partition getPartition(Hive db, ASTNode node, Table table) + throws SemanticException { // if this node has only one child, then no partition spec specified. if (node.getChildCount() == 1) { return null; @@ -149,43 +145,33 @@ private Map getPartitionSpec(Hive db, ASTNode node, TableName ta if (node.getChild(1).getType() == HiveParser.TOK_PARTSPEC) { ASTNode partNode = (ASTNode) node.getChild(1); - Table tab = null; + Map partitionSpec; try { - tab = db.getTable(tableName.getNotEmptyDbTable()); - } catch (InvalidTableException e) { - throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName.getNotEmptyDbTable()), e); - } catch (HiveException e) { - throw new SemanticException(e.getMessage(), e); + partitionSpec = getValidatedPartSpec(table, partNode, db.getConf(), false); + } catch (SemanticException e) { + // continue processing for DESCRIBE table key + return null; + } + + if (partitionSpec == null || partitionSpec.isEmpty()) { + return null; } - Map partitionSpec = null; + Partition partition; try { - partitionSpec = getPartSpec(partNode); - validateUnsupportedPartitionClause(tab, partitionSpec != null && !partitionSpec.isEmpty()); - partitionSpec = getValidatedPartSpec(tab, partNode, db.getConf(), false); - } catch (SemanticException e) { - // get exception in resolving partition it could be DESCRIBE table key - // return null, continue processing for DESCRIBE table key + partition = db.getPartition(table, partitionSpec); + } catch (HiveException e) { + // continue processing for DESCRIBE table key return null; } - if (partitionSpec != null) { - Partition part = null; - try { - part = db.getPartition(tab, partitionSpec, false); - } catch (HiveException e) { - // if get exception in finding partition it could be DESCRIBE table key - // return null, continue processing for DESCRIBE table key - return null; - } - - if (part == null) { - throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partitionSpec.toString())); - } - - return partitionSpec; + if (partition == null) { + throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partitionSpec.toString())); } + + return partition; } + // If child(1) is not TOK_PARTSPEC, it's a column name, return null return null; } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java index 0354a047a901..f0d979e80474 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -55,16 +56,16 @@ public class DescTableDesc implements DDLDesc, Serializable { private final String resFile; private final TableName tableName; - private final Map partitionSpec; + private final Partition partition; private final String columnPath; private final boolean isExtended; private final boolean isFormatted; - public DescTableDesc(Path resFile, TableName tableName, Map partitionSpec, String columnPath, + public DescTableDesc(Path resFile, TableName tableName, Partition partition, String columnPath, boolean isExtended, boolean isFormatted) { this.resFile = resFile.toString(); this.tableName = tableName; - this.partitionSpec = partitionSpec; + this.partition = partition; this.columnPath = columnPath; this.isExtended = isExtended; this.isFormatted = isFormatted; @@ -84,9 +85,13 @@ public TableName getTableName() { return tableName; } + public Partition getPartition() { + return partition; + } + @Explain(displayName = "partition", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public Map getPartitionSpec() { - return partitionSpec; + return partition != null ? partition.getSpec() : null; } public String getColumnPath() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java index 9f9f9351ef48..6401f9a8d9b4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java @@ -26,7 +26,6 @@ import java.util.Map; import com.google.common.primitives.Longs; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.ValidTxnList; @@ -75,7 +74,7 @@ public DescTableOperation(DDLOperationContext context, DescTableDesc desc) { @Override public int execute() throws Exception { Table table = getTable(); - Partition part = getPartition(table); + Partition part = desc.getPartition(); final String dbTableName = desc.getDbTableName(); try (DataOutputStream outStream = ShowUtils.getOutputStream(new Path(desc.getResFile()), context)) { @@ -124,18 +123,6 @@ private Table getTable() throws HiveException { return table; } - private Partition getPartition(Table table) throws HiveException { - Partition part = null; - if (desc.getPartitionSpec() != null) { - part = context.getDb().getPartition(table, desc.getPartitionSpec(), false); - if (part == null) { - throw new HiveException(ErrorMsg.INVALID_PARTITION, - StringUtils.join(desc.getPartitionSpec().keySet(), ','), desc.getDbTableName()); - } - } - return part; - } - private Deserializer getDeserializer(Table table) throws SQLException { return table.getDeserializer(true); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 6f73c36b6ad1..b1dd9738572a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -171,7 +171,13 @@ private void addPartitionData(DataOutputStream out, HiveConf conf, String column boolean isFormatted, boolean isOutputPadded) throws IOException { String partitionData = ""; if (columnPath == null) { - List partitionColumns = table.isPartitioned() ? table.getPartCols() : null; + List partitionColumns = null; + // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation + if (table.isPartitioned()) { + partitionColumns = table.hasNonNativePartitionSupport() ? + table.getStorageHandler().getPartitionKeys(table) : + table.getPartCols(); + } if (CollectionUtils.isNotEmpty(partitionColumns) && conf.getBoolVar(ConfVars.HIVE_DISPLAY_PARTITION_COLUMNS_SEPARATELY)) { TextMetaDataTable metaDataTable = new TextMetaDataTable(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java index 89681572c952..c188eb09fdcf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java @@ -38,24 +38,27 @@ public class DummyPartition extends Partition { private String name; private LinkedHashMap partSpec; + private List values; + public DummyPartition() { } - public DummyPartition(Table tbl, String name) { - setTable(tbl); + public DummyPartition(Table table, String name) { + setTable(table); this.name = name; } - public DummyPartition(Table tbl) { - this(tbl, null, Maps.newHashMap()); + public DummyPartition(Table table) { + this(table, null, Maps.newHashMap()); } - public DummyPartition(Table tbl, String name, Map partSpec) { - this(tbl, name); + public DummyPartition(Table table, String name, Map partSpec) { + this(table, name); org.apache.hadoop.hive.metastore.api.Partition tPart = new org.apache.hadoop.hive.metastore.api.Partition(); - tPart.setSd(tbl.getSd().deepCopy()); + tPart.setSd(table.getSd().deepCopy()); tPart.setParameters(Maps.newHashMap()); + tPart.setDbName(table.getDbName()); this.partSpec = Maps.newLinkedHashMap(partSpec); setTPartition(tPart); @@ -80,10 +83,21 @@ public LinkedHashMap getSpec() { @Override public List getValues() { - List values = new ArrayList(); - for (FieldSchema fs : this.getTable().getPartCols()) { - values.add(partSpec.get(fs.getName())); + if (values != null) { + return values; } + + Table table = this.getTable(); + values = new ArrayList<>(); + + // TODO (HIVE-29413): Refactor to a generic getPartCols() implementation + for (FieldSchema fs : table.hasNonNativePartitionSupport() + ? table.getStorageHandler().getPartitionKeys(table) + : table.getPartCols()) { + String val = partSpec.get(fs.getName()); + values.add(val); + } + return values; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java index 180431206109..a0a0f7500806 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java @@ -1748,23 +1748,6 @@ public static void validatePartSpec(Table tbl, Map partSpec, } } - /** - * Throws an UnsupportedOperationException in case the query has a partition clause but the table is never partitioned - * on the HMS-level. Even though table is not partitioned from the HMS's point of view, it might have some other - * notion of partitioning under the hood (e.g. Iceberg tables). In these cases, we might decide to proactively throw a - * more descriptive, unified error message instead of failing on some other semantic analysis validation step, which - * could provide a more counter-intuitive exception message. - * - * @param tbl The table object, should not be null. - * @param partitionClausePresent Whether a partition clause is present in the query (e.g. PARTITION(last_name='Don')) - */ - protected static void validateUnsupportedPartitionClause(Table tbl, boolean partitionClausePresent) { - if (partitionClausePresent && tbl.hasNonNativePartitionSupport()) { - throw new UnsupportedOperationException("Using partition spec in query is unsupported for non-native table" + - " backed by: " + tbl.getStorageHandler().toString()); - } - } - public static void validatePartColumnType(Table tbl, Map partSpec, ASTNode astNode, HiveConf conf) throws SemanticException { if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_TYPE_CHECK_ON_INSERT)) {