From 106d5d97a2d7a2f1bac70d73ea4dbfce4a16620d Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Mon, 11 Aug 2025 09:35:18 -0700 Subject: [PATCH 01/24] add wildcard support for rename Signed-off-by: Ritvi Bhatt --- .../org/opensearch/sql/analysis/Analyzer.java | 83 +++++++-- .../sql/utils/WildcardRenameUtils.java | 162 ++++++++++++++++++ ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + 3 files changed, 236 insertions(+), 10 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 719818e4c78..aca17db2b6d 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.util.Set; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -94,6 +95,7 @@ import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; @@ -133,6 +135,7 @@ import org.opensearch.sql.planner.physical.datasource.DataSourceTable; import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.ParseUtils; +import org.opensearch.sql.utils.WildcardRenameUtils; /** * Analyze the {@link UnresolvedPlan} in the {@link AnalysisContext} to construct the {@link @@ -306,20 +309,27 @@ private void verifySupportsCondition(Expression condition) { @Override public LogicalPlan visitRename(Rename node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); + + // Get available fields from current schema + Set availableFields = getAvailableFieldNames(context); + ImmutableMap.Builder renameMapBuilder = new ImmutableMap.Builder<>(); for (Map renameMap : node.getRenameList()) { - Expression origin = expressionAnalyzer.analyze(renameMap.getOrigin(), context); - // We should define the new target field in the context instead of analyze it. + String originPattern = ((Field) renameMap.getOrigin()).getField().toString(); + if (renameMap.getTarget() instanceof Field) { - ReferenceExpression target = - new ReferenceExpression( - ((Field) renameMap.getTarget()).getField().toString(), origin.type()); - ReferenceExpression originExpr = DSL.ref(origin.toString(), origin.type()); - TypeEnvironment curEnv = context.peek(); - curEnv.remove(originExpr); - curEnv.define(target); - renameMapBuilder.put(originExpr, target); + String targetPattern = ((Field) renameMap.getTarget()).getField().toString(); + + // Validate pattern compatibility for wildcards + if (WildcardRenameUtils.isWildcardPattern(originPattern) && + !WildcardRenameUtils.validatePatternCompatibility(originPattern, targetPattern)) { + throw new SemanticCheckException(String.format( + "Wildcard count mismatch between source '%s' and target '%s' patterns", + originPattern, targetPattern)); + } + + expandWildcardRename(originPattern, targetPattern, availableFields, renameMapBuilder, context); } else { throw new SemanticCheckException( String.format("the target expected to be field, but is %s", renameMap.getTarget())); @@ -895,4 +905,57 @@ private Aggregation analyzePatternsAgg(Patterns node) { groupByList.addAll(node.getPartitionByList()); return new Aggregation(aggExprs, ImmutableList.of(), groupByList); } + + /** + * Get available field names from current type environment. + * + * @param context the analysis context + * @return set of available field names + */ + private Set getAvailableFieldNames(AnalysisContext context) { + TypeEnvironment currentEnv = context.peek(); + return currentEnv.lookupAllFields(Namespace.FIELD_NAME).keySet(); + } + + /** + * Expand wildcard rename patterns to concrete field mappings. + * + * @param sourcePattern the source wildcard pattern + * @param targetPattern the target wildcard pattern + * @param availableFields set of available field names + * @param renameMapBuilder builder for rename mappings + * @param context the analysis context + */ + private void expandWildcardRename( + String sourcePattern, + String targetPattern, + Set availableFields, + ImmutableMap.Builder renameMapBuilder, + AnalysisContext context) { + + List matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); + + if (matchingFields.isEmpty()) { + throw new SemanticCheckException( + String.format("No fields match the wildcard pattern '%s'", sourcePattern)); + } + + TypeEnvironment curEnv = context.peek(); + + for (String fieldName : matchingFields) { + String newName = WildcardRenameUtils.applyWildcardTransformation( + sourcePattern, targetPattern, fieldName); + + Symbol fieldSymbol = new Symbol(Namespace.FIELD_NAME, fieldName); + ExprType fieldType = curEnv.resolve(fieldSymbol); + + ReferenceExpression origin = DSL.ref(fieldName, fieldType); + ReferenceExpression target = new ReferenceExpression(newName, fieldType); + + curEnv.remove(origin); + curEnv.define(target); + + renameMapBuilder.put(origin, target); + } + } } diff --git a/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java new file mode 100644 index 00000000000..753f83921f7 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java @@ -0,0 +1,162 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.utils; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Utility class for handling wildcard patterns in rename operations. + * Supports shell-style (*) wildcards. + */ +public class WildcardRenameUtils { + + /** + * Check if pattern contains any supported wildcards. + * + * @param pattern the pattern to check + * @return true if pattern contains * wildcards + */ + public static boolean isWildcardPattern(String pattern) { + return pattern.contains("*"); + } + + /** + * Check if pattern is a single wildcard that matches all fields. + * + * @param pattern the pattern to check + * @return true if pattern is exactly "*" + */ + public static boolean isFullWildcardPattern(String pattern) { + return "*".equals(pattern); + } + + + /** + * Convert wildcard pattern to regex with capture groups. + * + * @param pattern the wildcard pattern + * @return regex pattern with capture groups + */ + public static String wildcardToRegex(String pattern) { + String[] parts = pattern.split("\\*", -1); + return Arrays.stream(parts) + .map(Pattern::quote) + .collect(Collectors.joining("(.*)")); + } + + /** + * Match field names against wildcard pattern. + * + * @param wildcardPattern the pattern to match against + * @param availableFields set of available field names + * @return list of matching field names, sorted + */ + public static List matchFieldNames(String wildcardPattern, Set availableFields) { + if (!isWildcardPattern(wildcardPattern)) { + // No wildcards + return availableFields.contains(wildcardPattern) + ? List.of(wildcardPattern) + : List.of(); + } + + if (isFullWildcardPattern(wildcardPattern)) { + // Single wildcard matches all available fields + return availableFields.stream() + .sorted() + .collect(Collectors.toList()); + } + + String regexPattern = "^" + wildcardToRegex(wildcardPattern) + "$"; + Pattern pattern = Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE); + + return availableFields.stream() + .filter(field -> pattern.matcher(field).matches()) + .sorted() + .collect(Collectors.toList()); + } + + /** + * Apply wildcard transformation to generate new field name. + * + * @param sourcePattern the source wildcard pattern + * @param targetPattern the target wildcard pattern + * @param actualFieldName the actual field name to transform + * @return transformed field name + * @throws IllegalArgumentException if patterns don't match or are invalid + */ + public static String applyWildcardTransformation( + String sourcePattern, + String targetPattern, + String actualFieldName) { + + // No wildcards in either pattern + if (!isWildcardPattern(sourcePattern) && !isWildcardPattern(targetPattern)) { + return targetPattern; + } + + // Both are full wildcards + if (isFullWildcardPattern(sourcePattern) && isFullWildcardPattern(targetPattern)) { + return actualFieldName; + } + + if (isFullWildcardPattern(sourcePattern)) { + // Replace * in target with the actual field name + return targetPattern.replace("*", actualFieldName); + } + + String sourceRegex = "^" + wildcardToRegex(sourcePattern) + "$"; + Pattern sourceP = Pattern.compile(sourceRegex, Pattern.CASE_INSENSITIVE); + Matcher matcher = sourceP.matcher(actualFieldName); + + if (!matcher.matches()) { + throw new IllegalArgumentException( + String.format("Field '%s' does not match pattern '%s'", + actualFieldName, sourcePattern)); + } + + String result = targetPattern; + + for (int i = 1; i <= matcher.groupCount(); i++) { + String capturedValue = matcher.group(i); + + int index = result.indexOf("*"); + if (index >= 0) { + result = result.substring(0, index) + capturedValue + result.substring(index + 1); + } + } + + return result; + } + + /** + * Validate that source and target patterns have matching wildcard counts. + * + * @param sourcePattern the source pattern + * @param targetPattern the target pattern + * @return true if patterns are compatible + */ + public static boolean validatePatternCompatibility(String sourcePattern, String targetPattern) { + int sourceWildcards = countWildcards(sourcePattern); + int targetWildcards = countWildcards(targetPattern); + return sourceWildcards == targetWildcards; + } + + /** + * Count the number of wildcards in a pattern. + * + * @param pattern the pattern to analyze + * @return number of wildcard characters + */ + private static int countWildcards(String pattern) { + return (int) pattern.chars().filter(ch -> ch == '*').count(); + } + +} \ No newline at end of file diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 5f1da7a4801..a62deacab8c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -1212,6 +1212,7 @@ tableIdent wildcard : ident (MODULE ident)* (MODULE)? + | STAR | SINGLE_QUOTE wildcard SINGLE_QUOTE | DOUBLE_QUOTE wildcard DOUBLE_QUOTE | BACKTICK wildcard BACKTICK From 6d396732c70221f0cd797d11dddd6c8b00b634e9 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 14 Aug 2025 03:14:41 -0700 Subject: [PATCH 02/24] fix calcite wildcard support and add tests Signed-off-by: Ritvi Bhatt --- .../org/opensearch/sql/analysis/Analyzer.java | 109 +++++++--------- .../sql/calcite/CalciteRelNodeVisitor.java | 41 +++++- .../sql/utils/WildcardRenameUtils.java | 36 ++---- .../opensearch/sql/analysis/AnalyzerTest.java | 40 ++++++ .../sql/utils/WildcardRenameUtilsTest.java | 118 ++++++++++++++++++ docs/user/ppl/cmd/rename.rst | 23 +++- .../calcite/remote/CalcitePPLRenameIT.java | 56 +++++++++ .../opensearch/sql/ppl/RenameCommandIT.java | 35 +++++- 8 files changed, 358 insertions(+), 100 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index aca17db2b6d..a887297e508 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -309,30 +309,62 @@ private void verifySupportsCondition(Expression condition) { @Override public LogicalPlan visitRename(Rename node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); + ImmutableMap.Builder renameMapBuilder = + new ImmutableMap.Builder<>(); - // Get available fields from current schema Set availableFields = getAvailableFieldNames(context); - ImmutableMap.Builder renameMapBuilder = - new ImmutableMap.Builder<>(); for (Map renameMap : node.getRenameList()) { - String originPattern = ((Field) renameMap.getOrigin()).getField().toString(); - - if (renameMap.getTarget() instanceof Field) { + if (renameMap.getOrigin() instanceof Field && + WildcardRenameUtils.isWildcardPattern(((Field) renameMap.getOrigin()).getField().toString())) { + String fieldName = ((Field) renameMap.getOrigin()).getField().toString(); String targetPattern = ((Field) renameMap.getTarget()).getField().toString(); - // Validate pattern compatibility for wildcards - if (WildcardRenameUtils.isWildcardPattern(originPattern) && - !WildcardRenameUtils.validatePatternCompatibility(originPattern, targetPattern)) { - throw new SemanticCheckException(String.format( - "Wildcard count mismatch between source '%s' and target '%s' patterns", - originPattern, targetPattern)); + if (!WildcardRenameUtils.validatePatternCompatibility(fieldName, targetPattern)) { + throw new SemanticCheckException("Source and target patterns have different wildcard counts"); + } + + // Handle wildcard rename + List matchingFields = WildcardRenameUtils.matchFieldNames(fieldName, availableFields); + + if (matchingFields.isEmpty()) { + throw new SemanticCheckException( + String.format("No fields match the pattern '%s'", fieldName)); } - expandWildcardRename(originPattern, targetPattern, availableFields, renameMapBuilder, context); + TypeEnvironment curEnv = context.peek(); + + for (String matchingField : matchingFields) { + String newName = WildcardRenameUtils.applyWildcardTransformation( + fieldName, targetPattern, matchingField); + + Symbol fieldSymbol = new Symbol(Namespace.FIELD_NAME, matchingField); + ExprType fieldType = curEnv.resolve(fieldSymbol); + + ReferenceExpression origin = DSL.ref(matchingField, fieldType); + ReferenceExpression target = new ReferenceExpression(newName, fieldType); + + curEnv.remove(origin); + curEnv.define(target); + + renameMapBuilder.put(origin, target); + } } else { - throw new SemanticCheckException( - String.format("the target expected to be field, but is %s", renameMap.getTarget())); + Expression origin = expressionAnalyzer.analyze(renameMap.getOrigin(), context); + // We should define the new target field in the context instead of analyze it. + if (renameMap.getTarget() instanceof Field) { + ReferenceExpression target = + new ReferenceExpression( + ((Field) renameMap.getTarget()).getField().toString(), origin.type()); + ReferenceExpression originExpr = DSL.ref(origin.toString(), origin.type()); + TypeEnvironment curEnv = context.peek(); + curEnv.remove(originExpr); + curEnv.define(target); + renameMapBuilder.put(originExpr, target); + } else { + throw new SemanticCheckException( + String.format("the target expected to be field, but is %s", renameMap.getTarget())); + } } } @@ -906,56 +938,9 @@ private Aggregation analyzePatternsAgg(Patterns node) { return new Aggregation(aggExprs, ImmutableList.of(), groupByList); } - /** - * Get available field names from current type environment. - * - * @param context the analysis context - * @return set of available field names - */ private Set getAvailableFieldNames(AnalysisContext context) { TypeEnvironment currentEnv = context.peek(); return currentEnv.lookupAllFields(Namespace.FIELD_NAME).keySet(); } - /** - * Expand wildcard rename patterns to concrete field mappings. - * - * @param sourcePattern the source wildcard pattern - * @param targetPattern the target wildcard pattern - * @param availableFields set of available field names - * @param renameMapBuilder builder for rename mappings - * @param context the analysis context - */ - private void expandWildcardRename( - String sourcePattern, - String targetPattern, - Set availableFields, - ImmutableMap.Builder renameMapBuilder, - AnalysisContext context) { - - List matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); - - if (matchingFields.isEmpty()) { - throw new SemanticCheckException( - String.format("No fields match the wildcard pattern '%s'", sourcePattern)); - } - - TypeEnvironment curEnv = context.peek(); - - for (String fieldName : matchingFields) { - String newName = WildcardRenameUtils.applyWildcardTransformation( - sourcePattern, targetPattern, fieldName); - - Symbol fieldSymbol = new Symbol(Namespace.FIELD_NAME, fieldName); - ExprType fieldType = curEnv.resolve(fieldSymbol); - - ReferenceExpression origin = DSL.ref(fieldName, fieldType); - ReferenceExpression target = new ReferenceExpression(newName, fieldType); - - curEnv.remove(origin); - curEnv.define(target); - - renameMapBuilder.put(origin, target); - } - } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 13e62e5df07..aaf932d45cb 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -125,6 +125,7 @@ import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.PPLFuncImpTable; import org.opensearch.sql.utils.ParseUtils; +import org.opensearch.sql.utils.WildcardRenameUtils; public class CalciteRelNodeVisitor extends AbstractNodeVisitor { @@ -413,9 +414,42 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { visitChildren(node, context); List originalNames = context.relBuilder.peek().getRowType().getFieldNames(); List newNames = new ArrayList<>(originalNames); + for (org.opensearch.sql.ast.expression.Map renameMap : node.getRenameList()) { - if (renameMap.getTarget() instanceof Field t) { - String newName = t.getField().toString(); + if (!(renameMap.getTarget() instanceof Field)) { + throw new SemanticCheckException( + String.format("the target expected to be field, but is %s", renameMap.getTarget())); + } + + if (renameMap.getOrigin() instanceof Field && + WildcardRenameUtils.isWildcardPattern(((Field) renameMap.getOrigin()).getField().toString())) { + String sourcePattern = ((Field) renameMap.getOrigin()).getField().toString(); + String targetPattern = ((Field) renameMap.getTarget()).getField().toString(); + + if (!WildcardRenameUtils.validatePatternCompatibility(sourcePattern, targetPattern)) { + throw new SemanticCheckException("Source and target patterns have different wildcard counts"); + } + + // Handle wildcard rename for Calcite + Set availableFields = new HashSet<>(originalNames); + List matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); + + if (matchingFields.isEmpty()) { + throw new SemanticCheckException( + String.format("No fields match the pattern '%s'", sourcePattern)); + } + + for (String fieldName : matchingFields) { + String newName = WildcardRenameUtils.applyWildcardTransformation( + sourcePattern, targetPattern, fieldName); + + int fieldIndex = originalNames.indexOf(fieldName); + if (fieldIndex >= 0) { + newNames.set(fieldIndex, newName); + } + } + } else { + String newName = ((Field) renameMap.getTarget()).getField().toString(); RexNode check = rexVisitor.analyze(renameMap.getOrigin(), context); if (check instanceof RexInputRef ref) { newNames.set(ref.getIndex(), newName); @@ -423,9 +457,6 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { throw new SemanticCheckException( String.format("the original field %s cannot be resolved", renameMap.getOrigin())); } - } else { - throw new SemanticCheckException( - String.format("the target expected to be field, but is %s", renameMap.getTarget())); } } context.relBuilder.rename(newNames); diff --git a/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java index 753f83921f7..26ddb12ee66 100644 --- a/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java @@ -5,6 +5,7 @@ package org.opensearch.sql.utils; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Set; @@ -14,7 +15,6 @@ /** * Utility class for handling wildcard patterns in rename operations. - * Supports shell-style (*) wildcards. */ public class WildcardRenameUtils { @@ -37,10 +37,9 @@ public static boolean isWildcardPattern(String pattern) { public static boolean isFullWildcardPattern(String pattern) { return "*".equals(pattern); } - - + /** - * Convert wildcard pattern to regex with capture groups. + * Convert wildcard pattern to regex. * * @param pattern the wildcard pattern * @return regex pattern with capture groups @@ -57,34 +56,24 @@ public static String wildcardToRegex(String pattern) { * * @param wildcardPattern the pattern to match against * @param availableFields set of available field names - * @return list of matching field names, sorted + * @return list of matching field names */ public static List matchFieldNames(String wildcardPattern, Set availableFields) { - if (!isWildcardPattern(wildcardPattern)) { - // No wildcards - return availableFields.contains(wildcardPattern) - ? List.of(wildcardPattern) - : List.of(); - } - + // Single wildcard matches all available fields if (isFullWildcardPattern(wildcardPattern)) { - // Single wildcard matches all available fields - return availableFields.stream() - .sorted() - .collect(Collectors.toList()); + return new ArrayList<>(availableFields); } String regexPattern = "^" + wildcardToRegex(wildcardPattern) + "$"; - Pattern pattern = Pattern.compile(regexPattern, Pattern.CASE_INSENSITIVE); + Pattern pattern = Pattern.compile(regexPattern); return availableFields.stream() .filter(field -> pattern.matcher(field).matches()) - .sorted() .collect(Collectors.toList()); } /** - * Apply wildcard transformation to generate new field name. + * Apply wildcard transformation to get new field name. * * @param sourcePattern the source wildcard pattern * @param targetPattern the target wildcard pattern @@ -97,11 +86,6 @@ public static String applyWildcardTransformation( String targetPattern, String actualFieldName) { - // No wildcards in either pattern - if (!isWildcardPattern(sourcePattern) && !isWildcardPattern(targetPattern)) { - return targetPattern; - } - // Both are full wildcards if (isFullWildcardPattern(sourcePattern) && isFullWildcardPattern(targetPattern)) { return actualFieldName; @@ -113,7 +97,7 @@ public static String applyWildcardTransformation( } String sourceRegex = "^" + wildcardToRegex(sourcePattern) + "$"; - Pattern sourceP = Pattern.compile(sourceRegex, Pattern.CASE_INSENSITIVE); + Pattern sourceP = Pattern.compile(sourceRegex); Matcher matcher = sourceP.matcher(actualFieldName); if (!matcher.matches()) { @@ -126,7 +110,7 @@ public static String applyWildcardTransformation( for (int i = 1; i <= matcher.groupCount(); i++) { String capturedValue = matcher.group(i); - + int index = result.indexOf("*"); if (index >= 0) { result = result.substring(0, index) + capturedValue + result.substring(index + 1); diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 47ea2e5c5d3..eac6e9c9b41 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -501,6 +501,46 @@ public void rename_to_invalid_expression() { exception.getMessage()); } + @Test + public void rename_with_wildcard() { + assertAnalyzeEqual( + LogicalPlanDSL.rename( + LogicalPlanDSL.relation("schema", table), + ImmutableMap.of( + DSL.ref("string_value", STRING), DSL.ref("str_value", STRING), + DSL.ref("string_null_value", STRING), DSL.ref("str_null_value", STRING), + DSL.ref("string_missing_value", STRING), DSL.ref("str_missing_value", STRING))), + AstDSL.rename( + AstDSL.relation("schema"), + AstDSL.map(AstDSL.field("string_*"), AstDSL.field("str_*")))); + } + + @Test + public void rename_wildcard_no_matching_fields() { + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> + analyze( + AstDSL.rename( + AstDSL.relation("schema"), + AstDSL.map(AstDSL.field("*xyz"), AstDSL.field("*_field"))))); + assertEquals("No fields match the pattern '*xyz'", exception.getMessage()); + } + + @Test + public void rename_wildcard_pattern_mismatch() { + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> + analyze( + AstDSL.rename( + AstDSL.relation("schema"), + AstDSL.map(AstDSL.field("*name"), AstDSL.field("*_*_field"))))); + assertEquals("Source and target patterns have different wildcard counts", exception.getMessage()); + } + @Test public void project_source() { assertAnalyzeEqual( diff --git a/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java new file mode 100644 index 00000000000..46246513af3 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class WildcardRenameUtilsTest { + + @Test + void testIsWildcardPattern() { + assertTrue(WildcardRenameUtils.isWildcardPattern("*name")); + assertTrue(WildcardRenameUtils.isWildcardPattern("prefix*suffix")); + assertTrue(WildcardRenameUtils.isWildcardPattern("*")); + assertFalse(WildcardRenameUtils.isWildcardPattern("name")); + assertFalse(WildcardRenameUtils.isWildcardPattern("")); + } + + @Test + void testIsFullWildcardPattern() { + assertTrue(WildcardRenameUtils.isFullWildcardPattern("*")); + assertFalse(WildcardRenameUtils.isFullWildcardPattern("*name")); + assertFalse(WildcardRenameUtils.isFullWildcardPattern("prefix*")); + assertFalse(WildcardRenameUtils.isFullWildcardPattern("name")); + } + + @Test + void testWildcardToRegex() { + assertEquals("\\Q\\E(.*)\\Qname\\E", WildcardRenameUtils.wildcardToRegex("*name")); + assertEquals("\\Qname\\E(.*)\\Q\\E", WildcardRenameUtils.wildcardToRegex("name*")); + assertEquals("\\Q\\E(.*)\\Q_\\E(.*)\\Q_field\\E", WildcardRenameUtils.wildcardToRegex("*_*_field")); + } + + @Test + void testMatchFieldNames() { + LinkedHashSet fields = new LinkedHashSet<>(); + fields.add("firstname"); + fields.add("lastname"); + fields.add("age"); + fields.add("address"); + fields.add("fullname"); + + List nameFields = WildcardRenameUtils.matchFieldNames("*name", fields); + assertEquals(List.of("firstname", "lastname", "fullname"), nameFields); + List firstFields = WildcardRenameUtils.matchFieldNames("first*", fields); + assertEquals(List.of("firstname"), firstFields); + + // Test full wildcard - matches all fields + List allFields = WildcardRenameUtils.matchFieldNames("*", fields); + assertEquals(List.of("firstname", "lastname", "age", "address", "fullname"), allFields); + + // Test no matches + List noMatch = WildcardRenameUtils.matchFieldNames("*xyz", fields); + assertTrue(noMatch.isEmpty()); + + // Test exact match (no wildcards) + List exactMatch = WildcardRenameUtils.matchFieldNames("age", fields); + assertEquals(List.of("age"), exactMatch); + List exactNoMatch = WildcardRenameUtils.matchFieldNames("xyz", fields); + assertTrue(exactNoMatch.isEmpty()); + } + + @Test + void testApplyWildcardTransformation() { + assertEquals("firstNAME", + WildcardRenameUtils.applyWildcardTransformation("*name", "*NAME", "firstname")); + assertEquals("FIRSTname", + WildcardRenameUtils.applyWildcardTransformation("first*", "FIRST*", "firstname")); + assertEquals("user_profile", + WildcardRenameUtils.applyWildcardTransformation("*_*_field", "*_*", "user_profile_field")); + assertEquals("prefixfirst", + WildcardRenameUtils.applyWildcardTransformation("*name", "prefix*", "firstname")); + + // Test full wildcard transformations + assertEquals("firstname", + WildcardRenameUtils.applyWildcardTransformation("*", "*", "firstname")); + assertEquals("new_firstname", + WildcardRenameUtils.applyWildcardTransformation("*", "new_*", "firstname")); + assertEquals("first", + WildcardRenameUtils.applyWildcardTransformation("*name", "*", "firstname")); + } + + @Test + void testApplyWildcardTransformationErrors() { + // Test pattern mismatch - field doesn't match source pattern + assertThrows(IllegalArgumentException.class, () -> + WildcardRenameUtils.applyWildcardTransformation("*name", "*NAME", "age")); + } + + @Test + void testValidatePatternCompatibility() { + // Valid patterns + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*name", "*NAME")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*_*", "*_*")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("prefix*suffix", "PREFIX*SUFFIX")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("name", "NAME")); + + // Valid full wildcard patterns + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "*")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "new_*")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "*_old")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("old_*", "*")); + + // Invalid patterns - mismatched wildcard counts + assertFalse(WildcardRenameUtils.validatePatternCompatibility("*name", "*_*")); + assertFalse(WildcardRenameUtils.validatePatternCompatibility("*_*", "*")); + } +} \ No newline at end of file diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index c942884248e..7ae6a819a8c 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -18,8 +18,8 @@ Syntax ============ rename AS ["," AS ]... -* source-field: mandatory. The name of the field you want to rename. -* field list: mandatory. The name you want to rename to. +* source-field: mandatory. The name of the field you want to rename. Supports wildcard patterns using ``*``. +* target-field: mandatory. The name you want to rename to. Must have same number of wildcards as the source. Example 1: Rename one field @@ -59,6 +59,25 @@ PPL query:: | 18 | null | +----+---------+ + +Example 3: Rename with wildcards +================================= + +The example shows renaming multiple fields using wildcard patterns. + +PPL query:: + + os> source=accounts | rename *name as *_name | fields first_name, last_name; + fetched rows / total rows = 4/4 + +------------+-----------+ + | first_name | last_name | + |------------|-----------| + | Amber | Duke | + | Hattie | Bond | + | Nanette | Bates | + | Dale | Adams | + +------------+-----------+ + Limitation ========== The ``rename`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index 0458e7e5a2a..f032997a989 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -159,4 +159,60 @@ public void testRenameWithBackticksInAgg() throws IOException { verifySchemaInOrder(result, schema("avg(`user_age`)", "double"), schema("country", "string")); verifyDataRows(result, rows(22.5, "Canada"), rows(50.0, "USA")); } + + @Test + public void testRenameWildcardFields() throws IOException { + JSONObject result = + executeQuery( + String.format("source = %s | rename *ame as *AME", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("nAME", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + } + + @Test + public void testRenameMultipleWildcardFields() throws IOException { + JSONObject result = + executeQuery( + String.format("source = %s | rename *nt* as *NT*", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("couNTry", "string"), + schema("year", "int"), + schema("moNTh", "int")); + } + + @Test + public void testRenameWildcardPrefix() throws IOException { + JSONObject result = + executeQuery( + String.format("source = %s | rename *me as new_*", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("new_na", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + } + + @Test + public void testRenameFullWildcard() throws IOException { + JSONObject result = + executeQuery( + String.format("source = %s | fields name, age | rename * as old_*", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("old_name", "string"), + schema("old_age", "int")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java index 1844b5e07d7..1af4b7adff0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java @@ -43,12 +43,37 @@ public void testRenameMultiField() throws IOException { verifyColumn(result, columnName("FIRSTNAME"), columnName("AGE")); } - @Ignore( - "Wildcard is unsupported yet. Enable once" - + " https://github.com/opensearch-project/sql/issues/787 is resolved.") @Test public void testRenameWildcardFields() throws IOException { - JSONObject result = executeQuery("source=" + TEST_INDEX_ACCOUNT + " | rename %name as %NAME"); - verifyColumn(result, columnPattern(".*name$")); + JSONObject result = executeQuery("source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | rename *name as *NAME"); + verifyColumn(result, columnName("firstNAME"), columnName("lastNAME")); + } + + @Test + public void testRenameMultipleWildcardFields() throws IOException { + JSONObject result = executeQuery( + "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname, age | rename *name as new_*"); + verifyColumn(result, columnName("new_first"), columnName("new_last"), columnName("age")); + } + + @Test + public void testRenameWildcardPrefix() throws IOException { + JSONObject result = executeQuery( + "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname, age | rename first* as FIRST*"); + verifyColumn(result, columnName("FIRSTname"), columnName("lastname"), columnName("age")); + } + + @Test + public void testRenameFullWildcard() throws IOException { + JSONObject result = executeQuery( + "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | rename * as old_*"); + verifyColumn(result, columnName("old_firstname"), columnName("old_lastname")); + } + + @Test + public void testRenameWildcardWithMultipleCaptures() throws IOException { + JSONObject result = executeQuery( + "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | rename *first* as *FIRST*"); + verifyColumn(result, columnName("FIRSTname"), columnName("lastname")); } } From 0235b295b7ba09b958fd604ef0f0ec7d40ccf1db Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 14 Aug 2025 09:27:16 -0700 Subject: [PATCH 03/24] fix formatting Signed-off-by: Ritvi Bhatt --- .../org/opensearch/sql/analysis/Analyzer.java | 34 ++++++++-------- .../sql/calcite/CalciteRelNodeVisitor.java | 30 ++++++++------ .../sql/utils/WildcardRenameUtils.java | 40 ++++++++----------- .../opensearch/sql/analysis/AnalyzerTest.java | 3 +- .../sql/utils/WildcardRenameUtilsTest.java | 40 +++++++++++-------- .../calcite/remote/CalcitePPLRenameIT.java | 17 +++----- .../opensearch/sql/ppl/RenameCommandIT.java | 36 +++++++++++------ 7 files changed, 106 insertions(+), 94 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index a887297e508..3004460408e 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -25,7 +25,6 @@ import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import java.util.Set; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -310,32 +309,36 @@ private void verifySupportsCondition(Expression condition) { public LogicalPlan visitRename(Rename node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); ImmutableMap.Builder renameMapBuilder = - new ImmutableMap.Builder<>(); - + new ImmutableMap.Builder<>(); + Set availableFields = getAvailableFieldNames(context); - + for (Map renameMap : node.getRenameList()) { - if (renameMap.getOrigin() instanceof Field && - WildcardRenameUtils.isWildcardPattern(((Field) renameMap.getOrigin()).getField().toString())) { + if (renameMap.getOrigin() instanceof Field + && WildcardRenameUtils.isWildcardPattern( + ((Field) renameMap.getOrigin()).getField().toString())) { String fieldName = ((Field) renameMap.getOrigin()).getField().toString(); String targetPattern = ((Field) renameMap.getTarget()).getField().toString(); - + if (!WildcardRenameUtils.validatePatternCompatibility(fieldName, targetPattern)) { - throw new SemanticCheckException("Source and target patterns have different wildcard counts"); + throw new SemanticCheckException( + "Source and target patterns have different wildcard counts"); } - + // Handle wildcard rename - List matchingFields = WildcardRenameUtils.matchFieldNames(fieldName, availableFields); + List matchingFields = + WildcardRenameUtils.matchFieldNames(fieldName, availableFields); if (matchingFields.isEmpty()) { throw new SemanticCheckException( - String.format("No fields match the pattern '%s'", fieldName)); + String.format("No fields match the pattern '%s'", fieldName)); } TypeEnvironment curEnv = context.peek(); for (String matchingField : matchingFields) { - String newName = WildcardRenameUtils.applyWildcardTransformation( + String newName = + WildcardRenameUtils.applyWildcardTransformation( fieldName, targetPattern, matchingField); Symbol fieldSymbol = new Symbol(Namespace.FIELD_NAME, matchingField); @@ -354,8 +357,8 @@ public LogicalPlan visitRename(Rename node, AnalysisContext context) { // We should define the new target field in the context instead of analyze it. if (renameMap.getTarget() instanceof Field) { ReferenceExpression target = - new ReferenceExpression( - ((Field) renameMap.getTarget()).getField().toString(), origin.type()); + new ReferenceExpression( + ((Field) renameMap.getTarget()).getField().toString(), origin.type()); ReferenceExpression originExpr = DSL.ref(origin.toString(), origin.type()); TypeEnvironment curEnv = context.peek(); curEnv.remove(originExpr); @@ -363,7 +366,7 @@ public LogicalPlan visitRename(Rename node, AnalysisContext context) { renameMapBuilder.put(originExpr, target); } else { throw new SemanticCheckException( - String.format("the target expected to be field, but is %s", renameMap.getTarget())); + String.format("the target expected to be field, but is %s", renameMap.getTarget())); } } } @@ -942,5 +945,4 @@ private Set getAvailableFieldNames(AnalysisContext context) { TypeEnvironment currentEnv = context.peek(); return currentEnv.lookupAllFields(Namespace.FIELD_NAME).keySet(); } - } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index aaf932d45cb..6209fa4738d 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -414,35 +414,39 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { visitChildren(node, context); List originalNames = context.relBuilder.peek().getRowType().getFieldNames(); List newNames = new ArrayList<>(originalNames); - + for (org.opensearch.sql.ast.expression.Map renameMap : node.getRenameList()) { if (!(renameMap.getTarget() instanceof Field)) { throw new SemanticCheckException( String.format("the target expected to be field, but is %s", renameMap.getTarget())); } - - if (renameMap.getOrigin() instanceof Field && - WildcardRenameUtils.isWildcardPattern(((Field) renameMap.getOrigin()).getField().toString())) { + + if (renameMap.getOrigin() instanceof Field + && WildcardRenameUtils.isWildcardPattern( + ((Field) renameMap.getOrigin()).getField().toString())) { String sourcePattern = ((Field) renameMap.getOrigin()).getField().toString(); String targetPattern = ((Field) renameMap.getTarget()).getField().toString(); - + if (!WildcardRenameUtils.validatePatternCompatibility(sourcePattern, targetPattern)) { - throw new SemanticCheckException("Source and target patterns have different wildcard counts"); + throw new SemanticCheckException( + "Source and target patterns have different wildcard counts"); } - + // Handle wildcard rename for Calcite Set availableFields = new HashSet<>(originalNames); - List matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); - + List matchingFields = + WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); + if (matchingFields.isEmpty()) { throw new SemanticCheckException( - String.format("No fields match the pattern '%s'", sourcePattern)); + String.format("No fields match the pattern '%s'", sourcePattern)); } - + for (String fieldName : matchingFields) { - String newName = WildcardRenameUtils.applyWildcardTransformation( + String newName = + WildcardRenameUtils.applyWildcardTransformation( sourcePattern, targetPattern, fieldName); - + int fieldIndex = originalNames.indexOf(fieldName); if (fieldIndex >= 0) { newNames.set(fieldIndex, newName); diff --git a/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java index 26ddb12ee66..167ca250d53 100644 --- a/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java @@ -13,11 +13,9 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -/** - * Utility class for handling wildcard patterns in rename operations. - */ +/** Utility class for handling wildcard patterns in rename operations. */ public class WildcardRenameUtils { - + /** * Check if pattern contains any supported wildcards. * @@ -27,7 +25,7 @@ public class WildcardRenameUtils { public static boolean isWildcardPattern(String pattern) { return pattern.contains("*"); } - + /** * Check if pattern is a single wildcard that matches all fields. * @@ -46,11 +44,9 @@ public static boolean isFullWildcardPattern(String pattern) { */ public static String wildcardToRegex(String pattern) { String[] parts = pattern.split("\\*", -1); - return Arrays.stream(parts) - .map(Pattern::quote) - .collect(Collectors.joining("(.*)")); + return Arrays.stream(parts).map(Pattern::quote).collect(Collectors.joining("(.*)")); } - + /** * Match field names against wildcard pattern. * @@ -63,15 +59,15 @@ public static List matchFieldNames(String wildcardPattern, Set a if (isFullWildcardPattern(wildcardPattern)) { return new ArrayList<>(availableFields); } - + String regexPattern = "^" + wildcardToRegex(wildcardPattern) + "$"; Pattern pattern = Pattern.compile(regexPattern); - + return availableFields.stream() .filter(field -> pattern.matcher(field).matches()) .collect(Collectors.toList()); } - + /** * Apply wildcard transformation to get new field name. * @@ -82,10 +78,8 @@ public static List matchFieldNames(String wildcardPattern, Set a * @throws IllegalArgumentException if patterns don't match or are invalid */ public static String applyWildcardTransformation( - String sourcePattern, - String targetPattern, - String actualFieldName) { - + String sourcePattern, String targetPattern, String actualFieldName) { + // Both are full wildcards if (isFullWildcardPattern(sourcePattern) && isFullWildcardPattern(targetPattern)) { return actualFieldName; @@ -99,11 +93,10 @@ public static String applyWildcardTransformation( String sourceRegex = "^" + wildcardToRegex(sourcePattern) + "$"; Pattern sourceP = Pattern.compile(sourceRegex); Matcher matcher = sourceP.matcher(actualFieldName); - + if (!matcher.matches()) { throw new IllegalArgumentException( - String.format("Field '%s' does not match pattern '%s'", - actualFieldName, sourcePattern)); + String.format("Field '%s' does not match pattern '%s'", actualFieldName, sourcePattern)); } String result = targetPattern; @@ -116,10 +109,10 @@ public static String applyWildcardTransformation( result = result.substring(0, index) + capturedValue + result.substring(index + 1); } } - + return result; } - + /** * Validate that source and target patterns have matching wildcard counts. * @@ -132,7 +125,7 @@ public static boolean validatePatternCompatibility(String sourcePattern, String int targetWildcards = countWildcards(targetPattern); return sourceWildcards == targetWildcards; } - + /** * Count the number of wildcards in a pattern. * @@ -142,5 +135,4 @@ public static boolean validatePatternCompatibility(String sourcePattern, String private static int countWildcards(String pattern) { return (int) pattern.chars().filter(ch -> ch == '*').count(); } - -} \ No newline at end of file +} diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index eac6e9c9b41..42aa3963ae4 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -538,7 +538,8 @@ public void rename_wildcard_pattern_mismatch() { AstDSL.rename( AstDSL.relation("schema"), AstDSL.map(AstDSL.field("*name"), AstDSL.field("*_*_field"))))); - assertEquals("Source and target patterns have different wildcard counts", exception.getMessage()); + assertEquals( + "Source and target patterns have different wildcard counts", exception.getMessage()); } @Test diff --git a/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java index 46246513af3..513c9722df1 100644 --- a/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java @@ -12,7 +12,6 @@ import java.util.LinkedHashSet; import java.util.List; -import java.util.Set; import org.junit.jupiter.api.Test; class WildcardRenameUtilsTest { @@ -25,7 +24,7 @@ void testIsWildcardPattern() { assertFalse(WildcardRenameUtils.isWildcardPattern("name")); assertFalse(WildcardRenameUtils.isWildcardPattern("")); } - + @Test void testIsFullWildcardPattern() { assertTrue(WildcardRenameUtils.isFullWildcardPattern("*")); @@ -38,7 +37,8 @@ void testIsFullWildcardPattern() { void testWildcardToRegex() { assertEquals("\\Q\\E(.*)\\Qname\\E", WildcardRenameUtils.wildcardToRegex("*name")); assertEquals("\\Qname\\E(.*)\\Q\\E", WildcardRenameUtils.wildcardToRegex("name*")); - assertEquals("\\Q\\E(.*)\\Q_\\E(.*)\\Q_field\\E", WildcardRenameUtils.wildcardToRegex("*_*_field")); + assertEquals( + "\\Q\\E(.*)\\Q_\\E(.*)\\Q_field\\E", WildcardRenameUtils.wildcardToRegex("*_*_field")); } @Test @@ -72,29 +72,35 @@ void testMatchFieldNames() { @Test void testApplyWildcardTransformation() { - assertEquals("firstNAME", + assertEquals( + "firstNAME", WildcardRenameUtils.applyWildcardTransformation("*name", "*NAME", "firstname")); - assertEquals("FIRSTname", - WildcardRenameUtils.applyWildcardTransformation("first*", "FIRST*", "firstname")); - assertEquals("user_profile", + assertEquals( + "FIRSTname", + WildcardRenameUtils.applyWildcardTransformation("first*", "FIRST*", "firstname")); + assertEquals( + "user_profile", WildcardRenameUtils.applyWildcardTransformation("*_*_field", "*_*", "user_profile_field")); - assertEquals("prefixfirst", + assertEquals( + "prefixfirst", WildcardRenameUtils.applyWildcardTransformation("*name", "prefix*", "firstname")); // Test full wildcard transformations - assertEquals("firstname", - WildcardRenameUtils.applyWildcardTransformation("*", "*", "firstname")); - assertEquals("new_firstname", + assertEquals( + "firstname", WildcardRenameUtils.applyWildcardTransformation("*", "*", "firstname")); + assertEquals( + "new_firstname", WildcardRenameUtils.applyWildcardTransformation("*", "new_*", "firstname")); - assertEquals("first", - WildcardRenameUtils.applyWildcardTransformation("*name", "*", "firstname")); + assertEquals( + "first", WildcardRenameUtils.applyWildcardTransformation("*name", "*", "firstname")); } @Test void testApplyWildcardTransformationErrors() { // Test pattern mismatch - field doesn't match source pattern - assertThrows(IllegalArgumentException.class, () -> - WildcardRenameUtils.applyWildcardTransformation("*name", "*NAME", "age")); + assertThrows( + IllegalArgumentException.class, + () -> WildcardRenameUtils.applyWildcardTransformation("*name", "*NAME", "age")); } @Test @@ -104,7 +110,7 @@ void testValidatePatternCompatibility() { assertTrue(WildcardRenameUtils.validatePatternCompatibility("*_*", "*_*")); assertTrue(WildcardRenameUtils.validatePatternCompatibility("prefix*suffix", "PREFIX*SUFFIX")); assertTrue(WildcardRenameUtils.validatePatternCompatibility("name", "NAME")); - + // Valid full wildcard patterns assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "*")); assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "new_*")); @@ -115,4 +121,4 @@ void testValidatePatternCompatibility() { assertFalse(WildcardRenameUtils.validatePatternCompatibility("*name", "*_*")); assertFalse(WildcardRenameUtils.validatePatternCompatibility("*_*", "*")); } -} \ No newline at end of file +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index f032997a989..41f00d2d33f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -163,8 +163,7 @@ public void testRenameWithBackticksInAgg() throws IOException { @Test public void testRenameWildcardFields() throws IOException { JSONObject result = - executeQuery( - String.format("source = %s | rename *ame as *AME", TEST_INDEX_STATE_COUNTRY)); + executeQuery(String.format("source = %s | rename *ame as *AME", TEST_INDEX_STATE_COUNTRY)); verifySchema( result, schema("nAME", "string"), @@ -178,8 +177,7 @@ public void testRenameWildcardFields() throws IOException { @Test public void testRenameMultipleWildcardFields() throws IOException { JSONObject result = - executeQuery( - String.format("source = %s | rename *nt* as *NT*", TEST_INDEX_STATE_COUNTRY)); + executeQuery(String.format("source = %s | rename *nt* as *NT*", TEST_INDEX_STATE_COUNTRY)); verifySchema( result, schema("name", "string"), @@ -193,8 +191,7 @@ public void testRenameMultipleWildcardFields() throws IOException { @Test public void testRenameWildcardPrefix() throws IOException { JSONObject result = - executeQuery( - String.format("source = %s | rename *me as new_*", TEST_INDEX_STATE_COUNTRY)); + executeQuery(String.format("source = %s | rename *me as new_*", TEST_INDEX_STATE_COUNTRY)); verifySchema( result, schema("new_na", "string"), @@ -209,10 +206,8 @@ public void testRenameWildcardPrefix() throws IOException { public void testRenameFullWildcard() throws IOException { JSONObject result = executeQuery( - String.format("source = %s | fields name, age | rename * as old_*", TEST_INDEX_STATE_COUNTRY)); - verifySchema( - result, - schema("old_name", "string"), - schema("old_age", "int")); + String.format( + "source = %s | fields name, age | rename * as old_*", TEST_INDEX_STATE_COUNTRY)); + verifySchema(result, schema("old_name", "string"), schema("old_age", "int")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java index 1af4b7adff0..c5db32e0024 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java @@ -7,12 +7,10 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.util.MatcherUtils.columnName; -import static org.opensearch.sql.util.MatcherUtils.columnPattern; import static org.opensearch.sql.util.MatcherUtils.verifyColumn; import java.io.IOException; import org.json.JSONObject; -import org.junit.Ignore; import org.junit.jupiter.api.Test; public class RenameCommandIT extends PPLIntegTestCase { @@ -45,35 +43,49 @@ public void testRenameMultiField() throws IOException { @Test public void testRenameWildcardFields() throws IOException { - JSONObject result = executeQuery("source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | rename *name as *NAME"); + JSONObject result = + executeQuery( + "source=" + + TEST_INDEX_ACCOUNT + + " | fields firstname, lastname | rename *name as *NAME"); verifyColumn(result, columnName("firstNAME"), columnName("lastNAME")); } @Test public void testRenameMultipleWildcardFields() throws IOException { - JSONObject result = executeQuery( - "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname, age | rename *name as new_*"); + JSONObject result = + executeQuery( + "source=" + + TEST_INDEX_ACCOUNT + + " | fields firstname, lastname, age | rename *name as new_*"); verifyColumn(result, columnName("new_first"), columnName("new_last"), columnName("age")); } - @Test + @Test public void testRenameWildcardPrefix() throws IOException { - JSONObject result = executeQuery( - "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname, age | rename first* as FIRST*"); + JSONObject result = + executeQuery( + "source=" + + TEST_INDEX_ACCOUNT + + " | fields firstname, lastname, age | rename first* as FIRST*"); verifyColumn(result, columnName("FIRSTname"), columnName("lastname"), columnName("age")); } @Test public void testRenameFullWildcard() throws IOException { - JSONObject result = executeQuery( - "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | rename * as old_*"); + JSONObject result = + executeQuery( + "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | rename * as old_*"); verifyColumn(result, columnName("old_firstname"), columnName("old_lastname")); } @Test public void testRenameWildcardWithMultipleCaptures() throws IOException { - JSONObject result = executeQuery( - "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | rename *first* as *FIRST*"); + JSONObject result = + executeQuery( + "source=" + + TEST_INDEX_ACCOUNT + + " | fields firstname, lastname | rename *first* as *FIRST*"); verifyColumn(result, columnName("FIRSTname"), columnName("lastname")); } } From 90ae92a13f4de93b44ceb7f2a1a6d95256eb9229 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 14 Aug 2025 09:52:31 -0700 Subject: [PATCH 04/24] add check to analyzer Signed-off-by: Ritvi Bhatt --- .../org/opensearch/sql/analysis/Analyzer.java | 26 +++++++++---------- .../sql/calcite/CalciteRelNodeVisitor.java | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 3004460408e..8ab4db78496 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -314,6 +314,11 @@ public LogicalPlan visitRename(Rename node, AnalysisContext context) { Set availableFields = getAvailableFieldNames(context); for (Map renameMap : node.getRenameList()) { + if (!(renameMap.getTarget() instanceof Field)) { + throw new SemanticCheckException( + String.format("the target expected to be field, but is %s", renameMap.getTarget())); + } + if (renameMap.getOrigin() instanceof Field && WildcardRenameUtils.isWildcardPattern( ((Field) renameMap.getOrigin()).getField().toString())) { @@ -355,19 +360,14 @@ public LogicalPlan visitRename(Rename node, AnalysisContext context) { } else { Expression origin = expressionAnalyzer.analyze(renameMap.getOrigin(), context); // We should define the new target field in the context instead of analyze it. - if (renameMap.getTarget() instanceof Field) { - ReferenceExpression target = - new ReferenceExpression( - ((Field) renameMap.getTarget()).getField().toString(), origin.type()); - ReferenceExpression originExpr = DSL.ref(origin.toString(), origin.type()); - TypeEnvironment curEnv = context.peek(); - curEnv.remove(originExpr); - curEnv.define(target); - renameMapBuilder.put(originExpr, target); - } else { - throw new SemanticCheckException( - String.format("the target expected to be field, but is %s", renameMap.getTarget())); - } + ReferenceExpression target = + new ReferenceExpression( + ((Field) renameMap.getTarget()).getField().toString(), origin.type()); + ReferenceExpression originExpr = DSL.ref(origin.toString(), origin.type()); + TypeEnvironment curEnv = context.peek(); + curEnv.remove(originExpr); + curEnv.define(target); + renameMapBuilder.put(originExpr, target); } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 6209fa4738d..a7ae74695d6 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -421,6 +421,7 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { String.format("the target expected to be field, but is %s", renameMap.getTarget())); } + // Handle wildcards if (renameMap.getOrigin() instanceof Field && WildcardRenameUtils.isWildcardPattern( ((Field) renameMap.getOrigin()).getField().toString())) { @@ -432,7 +433,6 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { "Source and target patterns have different wildcard counts"); } - // Handle wildcard rename for Calcite Set availableFields = new HashSet<>(originalNames); List matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); From 9a893fa857a5191838fae0d8817fc1817b538416 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 14 Aug 2025 11:11:42 -0700 Subject: [PATCH 05/24] update doc formatting Signed-off-by: Ritvi Bhatt --- docs/user/ppl/cmd/rename.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index 7ae6a819a8c..0968e96911b 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -71,7 +71,7 @@ PPL query:: fetched rows / total rows = 4/4 +------------+-----------+ | first_name | last_name | - |------------|-----------| + |------------+-----------| | Amber | Duke | | Hattie | Bond | | Nanette | Bates | From a7b77db8ad1730f8451fdd34f64847560e7ef6c7 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Wed, 20 Aug 2025 00:28:53 -0700 Subject: [PATCH 06/24] remove v2 engine wildcard support Signed-off-by: Ritvi Bhatt --- .../org/opensearch/sql/analysis/Analyzer.java | 63 ++----------------- .../sql/calcite/CalciteRelNodeVisitor.java | 13 ++-- .../opensearch/sql/analysis/AnalyzerTest.java | 41 ------------ .../sql/utils/WildcardRenameUtilsTest.java | 5 ++ docs/user/ppl/cmd/rename.rst | 21 +++++++ .../calcite/remote/CalcitePPLRenameIT.java | 45 +++++++++++++ .../opensearch/sql/ppl/RenameCommandIT.java | 48 -------------- 7 files changed, 84 insertions(+), 152 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 8ab4db78496..8c8fbd5404b 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -31,7 +31,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -94,7 +93,6 @@ import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; @@ -134,7 +132,6 @@ import org.opensearch.sql.planner.physical.datasource.DataSourceTable; import org.opensearch.sql.storage.Table; import org.opensearch.sql.utils.ParseUtils; -import org.opensearch.sql.utils.WildcardRenameUtils; /** * Analyze the {@link UnresolvedPlan} in the {@link AnalysisContext} to construct the {@link @@ -310,56 +307,10 @@ public LogicalPlan visitRename(Rename node, AnalysisContext context) { LogicalPlan child = node.getChild().get(0).accept(this, context); ImmutableMap.Builder renameMapBuilder = new ImmutableMap.Builder<>(); - - Set availableFields = getAvailableFieldNames(context); - for (Map renameMap : node.getRenameList()) { - if (!(renameMap.getTarget() instanceof Field)) { - throw new SemanticCheckException( - String.format("the target expected to be field, but is %s", renameMap.getTarget())); - } - - if (renameMap.getOrigin() instanceof Field - && WildcardRenameUtils.isWildcardPattern( - ((Field) renameMap.getOrigin()).getField().toString())) { - String fieldName = ((Field) renameMap.getOrigin()).getField().toString(); - String targetPattern = ((Field) renameMap.getTarget()).getField().toString(); - - if (!WildcardRenameUtils.validatePatternCompatibility(fieldName, targetPattern)) { - throw new SemanticCheckException( - "Source and target patterns have different wildcard counts"); - } - - // Handle wildcard rename - List matchingFields = - WildcardRenameUtils.matchFieldNames(fieldName, availableFields); - - if (matchingFields.isEmpty()) { - throw new SemanticCheckException( - String.format("No fields match the pattern '%s'", fieldName)); - } - - TypeEnvironment curEnv = context.peek(); - - for (String matchingField : matchingFields) { - String newName = - WildcardRenameUtils.applyWildcardTransformation( - fieldName, targetPattern, matchingField); - - Symbol fieldSymbol = new Symbol(Namespace.FIELD_NAME, matchingField); - ExprType fieldType = curEnv.resolve(fieldSymbol); - - ReferenceExpression origin = DSL.ref(matchingField, fieldType); - ReferenceExpression target = new ReferenceExpression(newName, fieldType); - - curEnv.remove(origin); - curEnv.define(target); - - renameMapBuilder.put(origin, target); - } - } else { - Expression origin = expressionAnalyzer.analyze(renameMap.getOrigin(), context); - // We should define the new target field in the context instead of analyze it. + Expression origin = expressionAnalyzer.analyze(renameMap.getOrigin(), context); + // We should define the new target field in the context instead of analyze it. + if (renameMap.getTarget() instanceof Field) { ReferenceExpression target = new ReferenceExpression( ((Field) renameMap.getTarget()).getField().toString(), origin.type()); @@ -368,6 +319,9 @@ public LogicalPlan visitRename(Rename node, AnalysisContext context) { curEnv.remove(originExpr); curEnv.define(target); renameMapBuilder.put(originExpr, target); + } else { + throw new SemanticCheckException( + String.format("the target expected to be field, but is %s", renameMap.getTarget())); } } @@ -940,9 +894,4 @@ private Aggregation analyzePatternsAgg(Patterns node) { groupByList.addAll(node.getPartitionByList()); return new Aggregation(aggExprs, ImmutableList.of(), groupByList); } - - private Set getAvailableFieldNames(AnalysisContext context) { - TypeEnvironment currentEnv = context.peek(); - return currentEnv.lookupAllFields(Namespace.FIELD_NAME).keySet(); - } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index a7ae74695d6..2901d497e5a 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -432,16 +432,11 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { throw new SemanticCheckException( "Source and target patterns have different wildcard counts"); } - + // Use current newNames (which includes previous renames) for pattern matching Set availableFields = new HashSet<>(originalNames); List matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); - if (matchingFields.isEmpty()) { - throw new SemanticCheckException( - String.format("No fields match the pattern '%s'", sourcePattern)); - } - for (String fieldName : matchingFields) { String newName = WildcardRenameUtils.applyWildcardTransformation( @@ -450,8 +445,14 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { int fieldIndex = originalNames.indexOf(fieldName); if (fieldIndex >= 0) { newNames.set(fieldIndex, newName); + } else { + throw new SemanticCheckException( + String.format("the wildcard matched field %s cannot be resolved", fieldName)); } } + + // Update the RelBuilder context immediately so subsequent renames can see the changes + context.relBuilder.rename(newNames); } else { String newName = ((Field) renameMap.getTarget()).getField().toString(); RexNode check = rexVisitor.analyze(renameMap.getOrigin(), context); diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 42aa3963ae4..47ea2e5c5d3 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -501,47 +501,6 @@ public void rename_to_invalid_expression() { exception.getMessage()); } - @Test - public void rename_with_wildcard() { - assertAnalyzeEqual( - LogicalPlanDSL.rename( - LogicalPlanDSL.relation("schema", table), - ImmutableMap.of( - DSL.ref("string_value", STRING), DSL.ref("str_value", STRING), - DSL.ref("string_null_value", STRING), DSL.ref("str_null_value", STRING), - DSL.ref("string_missing_value", STRING), DSL.ref("str_missing_value", STRING))), - AstDSL.rename( - AstDSL.relation("schema"), - AstDSL.map(AstDSL.field("string_*"), AstDSL.field("str_*")))); - } - - @Test - public void rename_wildcard_no_matching_fields() { - SemanticCheckException exception = - assertThrows( - SemanticCheckException.class, - () -> - analyze( - AstDSL.rename( - AstDSL.relation("schema"), - AstDSL.map(AstDSL.field("*xyz"), AstDSL.field("*_field"))))); - assertEquals("No fields match the pattern '*xyz'", exception.getMessage()); - } - - @Test - public void rename_wildcard_pattern_mismatch() { - SemanticCheckException exception = - assertThrows( - SemanticCheckException.class, - () -> - analyze( - AstDSL.rename( - AstDSL.relation("schema"), - AstDSL.map(AstDSL.field("*name"), AstDSL.field("*_*_field"))))); - assertEquals( - "Source and target patterns have different wildcard counts", exception.getMessage()); - } - @Test public void project_source() { assertAnalyzeEqual( diff --git a/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java index 513c9722df1..62f51366378 100644 --- a/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java @@ -93,6 +93,11 @@ void testApplyWildcardTransformation() { WildcardRenameUtils.applyWildcardTransformation("*", "new_*", "firstname")); assertEquals( "first", WildcardRenameUtils.applyWildcardTransformation("*name", "*", "firstname")); + + // Test partial match + assertEquals( + "FiRsTname", + WildcardRenameUtils.applyWildcardTransformation("f*r*tname", "F*R*Tname", "firstname")); } @Test diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index 0968e96911b..1cd24895c04 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -21,6 +21,8 @@ rename AS ["," AS ]... * source-field: mandatory. The name of the field you want to rename. Supports wildcard patterns using ``*``. * target-field: mandatory. The name you want to rename to. Must have same number of wildcards as the source. +**Note:** Literal asterisk (*) characters in field names cannot be replaced as asterisk is used for wildcard matching. + Example 1: Rename one field =========================== @@ -78,6 +80,25 @@ PPL query:: | Dale | Adams | +------------+-----------+ + +Example 4: Rename with multiple wildcard patterns +================================================== + +The example shows renaming multiple fields using multiple wildcard patterns. + +PPL query:: + + os> source=accounts | rename *name as *_name, *_number as *number | fields first_name, last_name, accountnumber; + fetched rows / total rows = 4/4 + +------------+-----------+---------------+ + | first_name | last_name | accountnumber | + |------------+-----------+---------------| + | Amber | Duke | 1 | + | Hattie | Bond | 6 | + | Nanette | Bates | 13 | + | Dale | Adams | 18 | + +------------+-----------+---------------+ + Limitation ========== The ``rename`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index 41f00d2d33f..04f45bb21a1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -172,6 +172,12 @@ public void testRenameWildcardFields() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); } @Test @@ -186,6 +192,12 @@ public void testRenameMultipleWildcardFields() throws IOException { schema("couNTry", "string"), schema("year", "int"), schema("moNTh", "int")); + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); } @Test @@ -200,6 +212,12 @@ public void testRenameWildcardPrefix() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); } @Test @@ -209,5 +227,32 @@ public void testRenameFullWildcard() throws IOException { String.format( "source = %s | fields name, age | rename * as old_*", TEST_INDEX_STATE_COUNTRY)); verifySchema(result, schema("old_name", "string"), schema("old_age", "int")); + verifyDataRows( + result, + rows("Jake", 70), + rows("Hello", 30), + rows("John", 25), + rows("Jane", 20)); + } + + @Test + public void testRenameMultipleWildcards() throws IOException { + JSONObject result = + executeQuery( + String.format("source = %s | rename m*n*h as M*N*H", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("MoNtH", "int")); + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java index c5db32e0024..d7f4a1ba8f3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java @@ -40,52 +40,4 @@ public void testRenameMultiField() throws IOException { TEST_INDEX_ACCOUNT)); verifyColumn(result, columnName("FIRSTNAME"), columnName("AGE")); } - - @Test - public void testRenameWildcardFields() throws IOException { - JSONObject result = - executeQuery( - "source=" - + TEST_INDEX_ACCOUNT - + " | fields firstname, lastname | rename *name as *NAME"); - verifyColumn(result, columnName("firstNAME"), columnName("lastNAME")); - } - - @Test - public void testRenameMultipleWildcardFields() throws IOException { - JSONObject result = - executeQuery( - "source=" - + TEST_INDEX_ACCOUNT - + " | fields firstname, lastname, age | rename *name as new_*"); - verifyColumn(result, columnName("new_first"), columnName("new_last"), columnName("age")); - } - - @Test - public void testRenameWildcardPrefix() throws IOException { - JSONObject result = - executeQuery( - "source=" - + TEST_INDEX_ACCOUNT - + " | fields firstname, lastname, age | rename first* as FIRST*"); - verifyColumn(result, columnName("FIRSTname"), columnName("lastname"), columnName("age")); - } - - @Test - public void testRenameFullWildcard() throws IOException { - JSONObject result = - executeQuery( - "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | rename * as old_*"); - verifyColumn(result, columnName("old_firstname"), columnName("old_lastname")); - } - - @Test - public void testRenameWildcardWithMultipleCaptures() throws IOException { - JSONObject result = - executeQuery( - "source=" - + TEST_INDEX_ACCOUNT - + " | fields firstname, lastname | rename *first* as *FIRST*"); - verifyColumn(result, columnName("FIRSTname"), columnName("lastname")); - } } From 5b1c62bee34074ede0161c673820b15c42cbee08 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Wed, 20 Aug 2025 00:56:42 -0700 Subject: [PATCH 07/24] update doc Signed-off-by: Ritvi Bhatt --- docs/user/ppl/cmd/rename.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index 1cd24895c04..f9b9da17434 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -21,7 +21,10 @@ rename AS ["," AS ]... * source-field: mandatory. The name of the field you want to rename. Supports wildcard patterns using ``*``. * target-field: mandatory. The name you want to rename to. Must have same number of wildcards as the source. -**Note:** Literal asterisk (*) characters in field names cannot be replaced as asterisk is used for wildcard matching. +**Notes:** + +* Literal asterisk (*) characters in field names cannot be replaced as asterisk is used for wildcard matching. +* Wildcards are only supported when the Calcite query engine is enabled. Example 1: Rename one field @@ -65,11 +68,11 @@ PPL query:: Example 3: Rename with wildcards ================================= -The example shows renaming multiple fields using wildcard patterns. +The example shows renaming multiple fields using wildcard patterns. (Requires Calcite query engine) PPL query:: - os> source=accounts | rename *name as *_name | fields first_name, last_name; + PPL> source=accounts | rename *name as *_name | fields first_name, last_name; fetched rows / total rows = 4/4 +------------+-----------+ | first_name | last_name | @@ -84,11 +87,11 @@ PPL query:: Example 4: Rename with multiple wildcard patterns ================================================== -The example shows renaming multiple fields using multiple wildcard patterns. +The example shows renaming multiple fields using multiple wildcard patterns. (Requires Calcite query engine) PPL query:: - os> source=accounts | rename *name as *_name, *_number as *number | fields first_name, last_name, accountnumber; + PPL> source=accounts | rename *name as *_name, *_number as *number | fields first_name, last_name, accountnumber; fetched rows / total rows = 4/4 +------------+-----------+---------------+ | first_name | last_name | accountnumber | From 60c87138053eb1e1d5ca64d579f9dce6bfa0f9e3 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Wed, 20 Aug 2025 09:41:04 -0700 Subject: [PATCH 08/24] fix formatting Signed-off-by: Ritvi Bhatt --- .../opensearch/sql/calcite/remote/CalcitePPLRenameIT.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index 04f45bb21a1..b1df59eb9df 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -227,12 +227,7 @@ public void testRenameFullWildcard() throws IOException { String.format( "source = %s | fields name, age | rename * as old_*", TEST_INDEX_STATE_COUNTRY)); verifySchema(result, schema("old_name", "string"), schema("old_age", "int")); - verifyDataRows( - result, - rows("Jake", 70), - rows("Hello", 30), - rows("John", 25), - rows("Jane", 20)); + verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); } @Test From d01927b3fc003a806dc9f3891bb0c430dde83bcb Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Wed, 20 Aug 2025 12:22:27 -0700 Subject: [PATCH 09/24] support cascading rename Signed-off-by: Ritvi Bhatt --- .../sql/calcite/CalciteRelNodeVisitor.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 2901d497e5a..aab3b1845fa 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -422,7 +422,7 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { } // Handle wildcards - if (renameMap.getOrigin() instanceof Field + if (renameMap.getOrigin() instanceof Field && WildcardRenameUtils.isWildcardPattern( ((Field) renameMap.getOrigin()).getField().toString())) { String sourcePattern = ((Field) renameMap.getOrigin()).getField().toString(); @@ -432,8 +432,7 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { throw new SemanticCheckException( "Source and target patterns have different wildcard counts"); } - // Use current newNames (which includes previous renames) for pattern matching - Set availableFields = new HashSet<>(originalNames); + Set availableFields = new HashSet<>(newNames); List matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); @@ -442,29 +441,27 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { WildcardRenameUtils.applyWildcardTransformation( sourcePattern, targetPattern, fieldName); - int fieldIndex = originalNames.indexOf(fieldName); + int fieldIndex = newNames.indexOf(fieldName); if (fieldIndex >= 0) { newNames.set(fieldIndex, newName); + context.relBuilder.rename(newNames); } else { throw new SemanticCheckException( String.format("the wildcard matched field %s cannot be resolved", fieldName)); } } - - // Update the RelBuilder context immediately so subsequent renames can see the changes - context.relBuilder.rename(newNames); } else { String newName = ((Field) renameMap.getTarget()).getField().toString(); RexNode check = rexVisitor.analyze(renameMap.getOrigin(), context); if (check instanceof RexInputRef ref) { newNames.set(ref.getIndex(), newName); + context.relBuilder.rename(newNames); } else { throw new SemanticCheckException( String.format("the original field %s cannot be resolved", renameMap.getOrigin())); } } } - context.relBuilder.rename(newNames); return context.relBuilder.peek(); } From 610485b28b74c8e166eaf1ec12fd94cfaec5b8fa Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Wed, 20 Aug 2025 12:56:25 -0700 Subject: [PATCH 10/24] update formatting Signed-off-by: Ritvi Bhatt --- .../java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index aab3b1845fa..49f3551677a 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -422,7 +422,7 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { } // Handle wildcards - if (renameMap.getOrigin() instanceof Field + if (renameMap.getOrigin() instanceof Field && WildcardRenameUtils.isWildcardPattern( ((Field) renameMap.getOrigin()).getField().toString())) { String sourcePattern = ((Field) renameMap.getOrigin()).getField().toString(); From a3f266760aa2a5fdbd499dfe32ac41e9b0fa9c00 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Wed, 20 Aug 2025 15:07:17 -0700 Subject: [PATCH 11/24] add cross cluster test Signed-off-by: Ritvi Bhatt --- .../opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 85ece656ddf..dccbd4bdfad 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -119,6 +119,11 @@ public void testRenameCommandWithMultiFields() { anonymize("source=t | rename f as g,h as i,j as k")); } + @Test + public void testRenameCommandWithWildcards() { + assertEquals("source=t | rename f* as g*", anonymize("source=t | rename f* as g*")); + } + @Test public void testStatsCommandWithByClause() { assertEquals("source=t | stats count(a) by b", anonymize("source=t | stats count(a) by b")); From a2ebe1268f72fc5628cd88b021612a48a0c27bcd Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Wed, 20 Aug 2025 15:22:24 -0700 Subject: [PATCH 12/24] add test for cascading rename Signed-off-by: Ritvi Bhatt --- .../sql/calcite/remote/CalcitePPLRenameIT.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index b1df59eb9df..f8f96ceeafe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -250,4 +250,16 @@ public void testRenameMultipleWildcards() throws IOException { rows("John", "Canada", "Ontario", 4, 2023, 25), rows("Jane", "Canada", "Quebec", 4, 2023, 20)); } + + @Test + public void testCascadingRename() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | rename name as user_name | rename user_name as final_name | fields" + + " final_name, age", + TEST_INDEX_STATE_COUNTRY)); + verifySchema(result, schema("final_name", "string"), schema("age", "int")); + verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); + } } From 86eff6183187f4d3331918ef0eb387e8b5e7f467 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Wed, 20 Aug 2025 16:04:00 -0700 Subject: [PATCH 13/24] fix formatting Signed-off-by: Ritvi Bhatt --- .../java/org/opensearch/sql/security/CrossClusterSearchIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java index af1c08e8611..e8ab7965a66 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java @@ -168,7 +168,7 @@ public void testMatchAllCrossClusterDescribeAllFields() throws IOException { columnName("IS_AUTOINCREMENT"), columnName("IS_GENERATEDCOLUMN")); } - + @Test public void testCrossClusterSortWithCount() throws IOException { JSONObject result = From 775bd852ba3be41ea3ed2f0a4bbf0a34c1eff312 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 21 Aug 2025 09:39:22 -0700 Subject: [PATCH 14/24] add test for cascading rename Signed-off-by: Ritvi Bhatt --- .../calcite/remote/CalcitePPLRenameIT.java | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index f8f96ceeafe..cbb4ad613be 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -251,15 +251,38 @@ public void testRenameMultipleWildcards() throws IOException { rows("Jane", "Canada", "Quebec", 4, 2023, 20)); } + @Test + public void testMultipleRenameWithWildcard() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | fields name, age | rename name as user_name | rename user_name as" + + " final_name", + TEST_INDEX_STATE_COUNTRY)); + verifySchema(result, schema("final_name", "string"), schema("age", "int")); + verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); + } + @Test public void testCascadingRename() throws IOException { JSONObject result = executeQuery( String.format( - "source = %s | rename name as user_name | rename user_name as final_name | fields" - + " final_name, age", + "source = %s | fields name, age | rename name as user_name, user_name as" + + " final_name", TEST_INDEX_STATE_COUNTRY)); verifySchema(result, schema("final_name", "string"), schema("age", "int")); verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); } + + @Test + public void testCascadingRenameWithWildcard() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | fields name, age | rename *ame as *_ame, *_ame as *_AME", + TEST_INDEX_STATE_COUNTRY)); + verifySchema(result, schema("n_AME", "string"), schema("age", "int")); + verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); + } } From 845820f5f1205d7bbbfcbfadf7dc32fe201a3bb4 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 21 Aug 2025 17:58:37 -0700 Subject: [PATCH 15/24] change behavior for renaming existing fields Signed-off-by: Ritvi Bhatt --- .../sql/calcite/CalciteRelNodeVisitor.java | 72 ++++++++------- .../calcite/remote/CalcitePPLRenameIT.java | 89 +++++++++++++++---- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 +- 3 files changed, 108 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 49f3551677a..5fa0a775e13 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -421,45 +421,43 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { String.format("the target expected to be field, but is %s", renameMap.getTarget())); } - // Handle wildcards - if (renameMap.getOrigin() instanceof Field - && WildcardRenameUtils.isWildcardPattern( - ((Field) renameMap.getOrigin()).getField().toString())) { - String sourcePattern = ((Field) renameMap.getOrigin()).getField().toString(); - String targetPattern = ((Field) renameMap.getTarget()).getField().toString(); - - if (!WildcardRenameUtils.validatePatternCompatibility(sourcePattern, targetPattern)) { - throw new SemanticCheckException( - "Source and target patterns have different wildcard counts"); - } - Set availableFields = new HashSet<>(newNames); - List matchingFields = - WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); - - for (String fieldName : matchingFields) { - String newName = - WildcardRenameUtils.applyWildcardTransformation( - sourcePattern, targetPattern, fieldName); - - int fieldIndex = newNames.indexOf(fieldName); - if (fieldIndex >= 0) { - newNames.set(fieldIndex, newName); - context.relBuilder.rename(newNames); - } else { - throw new SemanticCheckException( - String.format("the wildcard matched field %s cannot be resolved", fieldName)); - } + String sourcePattern = ((Field) renameMap.getOrigin()).getField().toString(); + String targetPattern = ((Field) renameMap.getTarget()).getField().toString(); + + if (WildcardRenameUtils.isWildcardPattern(sourcePattern) + && !WildcardRenameUtils.validatePatternCompatibility(sourcePattern, targetPattern)) { + throw new SemanticCheckException( + "Source and target patterns have different wildcard counts"); + } + + Set availableFields = new HashSet<>(newNames); + List matchingFields = + WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); + + for (String fieldName : matchingFields) { + String newName; + newName = + WildcardRenameUtils.applyWildcardTransformation( + sourcePattern, targetPattern, fieldName); + + // If target field already exists, remove field before renaming source + int existingFieldIndex = newNames.indexOf(newName); + if (existingFieldIndex != -1) { + newNames.remove(newName); + context.relBuilder.projectExcept(context.relBuilder.field(newName)); } - } else { - String newName = ((Field) renameMap.getTarget()).getField().toString(); - RexNode check = rexVisitor.analyze(renameMap.getOrigin(), context); - if (check instanceof RexInputRef ref) { - newNames.set(ref.getIndex(), newName); - context.relBuilder.rename(newNames); - } else { - throw new SemanticCheckException( - String.format("the original field %s cannot be resolved", renameMap.getOrigin())); + + int fieldIndex = newNames.indexOf(fieldName); + if (fieldIndex != -1) { + newNames.set(fieldIndex, newName); } + context.relBuilder.rename(newNames); + } + // if source field doesn't exist but target does, remove target field from results + if (matchingFields.isEmpty() && !WildcardRenameUtils.isWildcardPattern(targetPattern)) { + newNames.remove(targetPattern); + context.relBuilder.projectExcept(context.relBuilder.field(targetPattern)); + context.relBuilder.rename(newNames); } } return context.relBuilder.peek(); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index cbb4ad613be..9586e4ece26 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -58,21 +58,6 @@ public void testRefRenamedField() { + " _id, _index, _score, _maxscore, _sort, _routing]"); } - @Test - public void testRenameNotExistedField() { - Throwable e = - assertThrowsWithReplace( - IllegalArgumentException.class, - () -> - executeQuery( - String.format( - "source = %s | rename renamed_age as age", TEST_INDEX_STATE_COUNTRY))); - verifyErrorMessageContains( - e, - "field [renamed_age] not found; input fields are: [name, country, state, month, year, age," - + " _id, _index, _score, _maxscore, _sort, _routing]"); - } - @Test public void testRenameToMetaField() throws IOException { Throwable e = @@ -264,7 +249,7 @@ public void testMultipleRenameWithWildcard() throws IOException { } @Test - public void testCascadingRename() throws IOException { + public void testChainedRename() throws IOException { JSONObject result = executeQuery( String.format( @@ -276,7 +261,7 @@ public void testCascadingRename() throws IOException { } @Test - public void testCascadingRenameWithWildcard() throws IOException { + public void testChainedRenameWithWildcard() throws IOException { JSONObject result = executeQuery( String.format( @@ -285,4 +270,74 @@ public void testCascadingRenameWithWildcard() throws IOException { verifySchema(result, schema("n_AME", "string"), schema("age", "int")); verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); } + + @Test + public void testRenamingToExistingField() throws IOException { + JSONObject result = + executeQuery(String.format("source = %s | rename name as age", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("age", "string"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyDataRows( + result, + rows("Jake", "California", "USA", 2023, 4), + rows("Hello", "New York", "USA", 2023, 4), + rows("John", "Ontario", "Canada", 2023, 4), + rows("Jane", "Quebec", "Canada", 2023, 4)); + } + + @Test + public void testRenamingNonExistentField() throws IOException { + JSONObject result = + executeQuery( + String.format("source = %s | rename none as nothing", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyDataRows( + result, + rows("Jake", 70, "California", "USA", 2023, 4), + rows("Hello", 30, "New York", "USA", 2023, 4), + rows("John", 25, "Ontario", "Canada", 2023, 4), + rows("Jane", 20, "Quebec", "Canada", 2023, 4)); + } + + @Test + public void testRenamingNonExistentFieldToExistingField() throws IOException { + JSONObject result = + executeQuery(String.format("source = %s | rename none as age", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyDataRows( + result, + rows("Jake", "California", "USA", 2023, 4), + rows("Hello", "New York", "USA", 2023, 4), + rows("John", "Ontario", "Canada", 2023, 4), + rows("Jane", "Quebec", "Canada", 2023, 4)); + } + + @Test + public void testWildcardPatternDifferentCounts() { + Throwable e = + assertThrowsWithReplace( + IllegalArgumentException.class, + () -> + executeQuery( + String.format("source = %s | rename *a*e as *new", TEST_INDEX_STATE_COUNTRY))); + verifyErrorMessageContains(e, "Source and target patterns have different wildcard counts"); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index a62deacab8c..9cb93826f7e 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -145,7 +145,7 @@ wcFieldList ; renameCommand - : RENAME renameClasue (COMMA renameClasue)* + : RENAME renameClasue (COMMA? renameClasue)* ; statsCommand From a4c08e1df2691ecd2ee05a74ba55a38150b50582 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 22 Aug 2025 11:34:54 -0700 Subject: [PATCH 16/24] add tests and update docs Signed-off-by: Ritvi Bhatt --- .../sql/calcite/CalciteRelNodeVisitor.java | 2 +- docs/user/ppl/cmd/rename.rst | 32 ++++++++++++++++- .../calcite/remote/CalcitePPLRenameIT.java | 36 ++++++++++++------- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 5fa0a775e13..2a44791f054 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -454,7 +454,7 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { context.relBuilder.rename(newNames); } // if source field doesn't exist but target does, remove target field from results - if (matchingFields.isEmpty() && !WildcardRenameUtils.isWildcardPattern(targetPattern)) { + if (matchingFields.isEmpty() && newNames.contains(targetPattern)) { newNames.remove(targetPattern); context.relBuilder.projectExcept(context.relBuilder.field(targetPattern)); context.relBuilder.rename(newNames); diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index f9b9da17434..aa292f8182e 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -18,9 +18,19 @@ Syntax ============ rename AS ["," AS ]... -* source-field: mandatory. The name of the field you want to rename. Supports wildcard patterns using ``*``. +* source-field: mandatory. The name of the field you want to rename. Supports wildcard patterns since version 3.3 using ``*``. * target-field: mandatory. The name you want to rename to. Must have same number of wildcards as the source. +Behavior with Non-existent Fields (Since version 3.3) +===================================================== + +The rename command handles non-existent fields as follows: + +* **Renaming a non-existent field to a non-existent field**: No change occurs to the result set. +* **Renaming a non-existent field to an existing field**: The existing target field is removed from the result set. +* **Renaming an existing field to an existing field**: The existing target field is removed first, then the source field is renamed to the target. + + **Notes:** * Literal asterisk (*) characters in field names cannot be replaced as asterisk is used for wildcard matching. @@ -102,6 +112,26 @@ PPL query:: | Dale | Adams | 18 | +------------+-----------+---------------+ +Example 5: Rename existing field to existing field +==================================== + +The example shows renaming an existing field to an existing field. The target field gets removed and the source field is renamed to the target field. + + +PPL query:: + + PPL> source=accounts | rename firstname as age | fields age; + fetched rows / total rows = 4/4 + +---------+ + | age | + |---------| + | Amber | + | Hattie | + | Nanette | + | Dale | + +---------+ + + Limitation ========== The ``rename`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index 9586e4ece26..36303a86683 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -40,6 +40,12 @@ public void testRename() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); } @Test @@ -79,6 +85,12 @@ public void testRenameToMetaField() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); } @Test @@ -284,10 +296,10 @@ public void testRenamingToExistingField() throws IOException { schema("month", "int")); verifyDataRows( result, - rows("Jake", "California", "USA", 2023, 4), - rows("Hello", "New York", "USA", 2023, 4), - rows("John", "Ontario", "Canada", 2023, 4), - rows("Jane", "Quebec", "Canada", 2023, 4)); + rows("Jake", "USA", "California", 4, 2023), + rows("Hello", "USA", "New York", 4, 2023), + rows("John", "Canada", "Ontario", 4, 2023), + rows("Jane", "Canada", "Quebec", 4, 2023)); } @Test @@ -305,10 +317,10 @@ public void testRenamingNonExistentField() throws IOException { schema("month", "int")); verifyDataRows( result, - rows("Jake", 70, "California", "USA", 2023, 4), - rows("Hello", 30, "New York", "USA", 2023, 4), - rows("John", 25, "Ontario", "Canada", 2023, 4), - rows("Jane", 20, "Quebec", "Canada", 2023, 4)); + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); } @Test @@ -324,10 +336,10 @@ public void testRenamingNonExistentFieldToExistingField() throws IOException { schema("month", "int")); verifyDataRows( result, - rows("Jake", "California", "USA", 2023, 4), - rows("Hello", "New York", "USA", 2023, 4), - rows("John", "Ontario", "Canada", 2023, 4), - rows("Jane", "Quebec", "Canada", 2023, 4)); + rows("Jake", "USA", "California", 4, 2023), + rows("Hello", "USA", "New York", 4, 2023), + rows("John", "Canada", "Ontario", 4, 2023), + rows("Jane", "Canada", "Quebec", 4, 2023)); } @Test From 6c51ffde21d76893c844ab3f5b7310df61d563ea Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 22 Aug 2025 12:11:54 -0700 Subject: [PATCH 17/24] update docs Signed-off-by: Ritvi Bhatt --- docs/user/ppl/cmd/rename.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index aa292f8182e..82fd6c982cb 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -21,8 +21,8 @@ rename AS ["," AS ]... * source-field: mandatory. The name of the field you want to rename. Supports wildcard patterns since version 3.3 using ``*``. * target-field: mandatory. The name you want to rename to. Must have same number of wildcards as the source. -Behavior with Non-existent Fields (Since version 3.3) -===================================================== +Field Rename Behavior (Since version 3.3) +========================================== The rename command handles non-existent fields as follows: From 58015426abe26bc0378c2b2001cff40d92beca89 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 22 Aug 2025 12:13:49 -0700 Subject: [PATCH 18/24] update docs Signed-off-by: Ritvi Bhatt --- docs/user/ppl/cmd/rename.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index 82fd6c982cb..9c171f697ed 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -28,7 +28,7 @@ The rename command handles non-existent fields as follows: * **Renaming a non-existent field to a non-existent field**: No change occurs to the result set. * **Renaming a non-existent field to an existing field**: The existing target field is removed from the result set. -* **Renaming an existing field to an existing field**: The existing target field is removed first, then the source field is renamed to the target. +* **Renaming an existing field to an existing field**: The existing target field is removed and the source field is renamed to the target. **Notes:** From 2d3a38b6a77a28a8d19b2c9b4941ab031d4a095f Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Fri, 22 Aug 2025 12:36:08 -0700 Subject: [PATCH 19/24] fix renaming to same name Signed-off-by: Ritvi Bhatt --- .../sql/calcite/CalciteRelNodeVisitor.java | 2 +- .../calcite/remote/CalcitePPLRenameIT.java | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 2a44791f054..5c93b408b7c 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -442,7 +442,7 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { // If target field already exists, remove field before renaming source int existingFieldIndex = newNames.indexOf(newName); - if (existingFieldIndex != -1) { + if (existingFieldIndex != -1 && !fieldName.equals(newName)) { newNames.remove(newName); context.relBuilder.projectExcept(context.relBuilder.field(newName)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index 36303a86683..b746a66d498 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -352,4 +352,24 @@ public void testWildcardPatternDifferentCounts() { String.format("source = %s | rename *a*e as *new", TEST_INDEX_STATE_COUNTRY))); verifyErrorMessageContains(e, "Source and target patterns have different wildcard counts"); } + + @Test + public void testRenameSameField() throws IOException { + JSONObject result = + executeQuery(String.format("source = %s | rename age as age", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + } } From b6fb2ba3cb8c58fef070b8eeed7f2c70cb9596d2 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Tue, 2 Sep 2025 16:40:07 -0700 Subject: [PATCH 20/24] fix behavior for consecutive wildcards/address comments Signed-off-by: Ritvi Bhatt --- .../sql/calcite/CalciteRelNodeVisitor.java | 29 ++--- .../sql/utils/WildcardRenameUtils.java | 29 +++-- .../sql/utils/WildcardRenameUtilsTest.java | 118 ++++++++++++++---- docs/user/ppl/cmd/rename.rst | 6 +- .../calcite/remote/CalcitePPLRenameIT.java | 89 ++++++------- .../security/CalciteCrossClusterSearchIT.java | 26 ++++ .../sql/security/CrossClusterSearchIT.java | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 8 +- .../sql/ppl/parser/AstExpressionBuilder.java | 9 ++ 9 files changed, 214 insertions(+), 102 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 5c93b408b7c..bcb74a26528 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -430,39 +430,36 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { "Source and target patterns have different wildcard counts"); } - Set availableFields = new HashSet<>(newNames); - List matchingFields = - WildcardRenameUtils.matchFieldNames(sourcePattern, availableFields); + List matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, newNames); for (String fieldName : matchingFields) { - String newName; - newName = + String newName = WildcardRenameUtils.applyWildcardTransformation( sourcePattern, targetPattern, fieldName); - - // If target field already exists, remove field before renaming source - int existingFieldIndex = newNames.indexOf(newName); - if (existingFieldIndex != -1 && !fieldName.equals(newName)) { - newNames.remove(newName); - context.relBuilder.projectExcept(context.relBuilder.field(newName)); + if (newNames.contains(newName) && !newName.equals(fieldName)) { + removeFieldIfExists(newName, newNames, context); } - int fieldIndex = newNames.indexOf(fieldName); if (fieldIndex != -1) { newNames.set(fieldIndex, newName); } - context.relBuilder.rename(newNames); } - // if source field doesn't exist but target does, remove target field from results + if (matchingFields.isEmpty() && newNames.contains(targetPattern)) { - newNames.remove(targetPattern); - context.relBuilder.projectExcept(context.relBuilder.field(targetPattern)); + removeFieldIfExists(targetPattern, newNames, context); context.relBuilder.rename(newNames); } } + context.relBuilder.rename(newNames); return context.relBuilder.peek(); } + private void removeFieldIfExists( + String fieldName, List newNames, CalcitePlanContext context) { + newNames.remove(fieldName); + context.relBuilder.projectExcept(context.relBuilder.field(fieldName)); + } + @Override public RelNode visitSort(Sort node, CalcitePlanContext context) { visitChildren(node, context); diff --git a/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java index 167ca250d53..36912baed2c 100644 --- a/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java @@ -7,8 +7,8 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; -import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -27,13 +27,13 @@ public static boolean isWildcardPattern(String pattern) { } /** - * Check if pattern is a single wildcard that matches all fields. + * Check if pattern is only wildcards that matches all fields. * * @param pattern the pattern to check - * @return true if pattern is exactly "*" + * @return true if pattern is only made up of wildcards "*" */ public static boolean isFullWildcardPattern(String pattern) { - return "*".equals(pattern); + return pattern.matches("\\*+"); } /** @@ -51,10 +51,11 @@ public static String wildcardToRegex(String pattern) { * Match field names against wildcard pattern. * * @param wildcardPattern the pattern to match against - * @param availableFields set of available field names + * @param availableFields collection of available field names * @return list of matching field names */ - public static List matchFieldNames(String wildcardPattern, Set availableFields) { + public static List matchFieldNames( + String wildcardPattern, Collection availableFields) { // Single wildcard matches all available fields if (isFullWildcardPattern(wildcardPattern)) { return new ArrayList<>(availableFields); @@ -80,19 +81,18 @@ public static List matchFieldNames(String wildcardPattern, Set a public static String applyWildcardTransformation( String sourcePattern, String targetPattern, String actualFieldName) { - // Both are full wildcards - if (isFullWildcardPattern(sourcePattern) && isFullWildcardPattern(targetPattern)) { + if (sourcePattern.equals(targetPattern)) { return actualFieldName; } - if (isFullWildcardPattern(sourcePattern)) { - // Replace * in target with the actual field name - return targetPattern.replace("*", actualFieldName); + if (!isFullWildcardPattern(sourcePattern) || !isFullWildcardPattern(targetPattern)) { + if (sourcePattern.matches(".*\\*{2,}.*") || targetPattern.matches(".*\\*{2,}.*")) { + throw new IllegalArgumentException("Consecutive wildcards in pattern are not supported"); + } } String sourceRegex = "^" + wildcardToRegex(sourcePattern) + "$"; - Pattern sourceP = Pattern.compile(sourceRegex); - Matcher matcher = sourceP.matcher(actualFieldName); + Matcher matcher = Pattern.compile(sourceRegex).matcher(actualFieldName); if (!matcher.matches()) { throw new IllegalArgumentException( @@ -107,6 +107,9 @@ public static String applyWildcardTransformation( int index = result.indexOf("*"); if (index >= 0) { result = result.substring(0, index) + capturedValue + result.substring(index + 1); + } else { + throw new IllegalArgumentException( + "Target pattern has fewer wildcards than source pattern"); } } diff --git a/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java index 62f51366378..a2582fa58f8 100644 --- a/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java @@ -10,7 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.LinkedHashSet; +import com.google.common.collect.ImmutableSet; import java.util.List; import org.junit.jupiter.api.Test; @@ -28,9 +28,12 @@ void testIsWildcardPattern() { @Test void testIsFullWildcardPattern() { assertTrue(WildcardRenameUtils.isFullWildcardPattern("*")); + assertTrue(WildcardRenameUtils.isFullWildcardPattern("**")); + assertTrue(WildcardRenameUtils.isFullWildcardPattern("***")); assertFalse(WildcardRenameUtils.isFullWildcardPattern("*name")); assertFalse(WildcardRenameUtils.isFullWildcardPattern("prefix*")); assertFalse(WildcardRenameUtils.isFullWildcardPattern("name")); + assertFalse(WildcardRenameUtils.isFullWildcardPattern("*_*")); } @Test @@ -43,30 +46,32 @@ void testWildcardToRegex() { @Test void testMatchFieldNames() { - LinkedHashSet fields = new LinkedHashSet<>(); - fields.add("firstname"); - fields.add("lastname"); - fields.add("age"); - fields.add("address"); - fields.add("fullname"); - - List nameFields = WildcardRenameUtils.matchFieldNames("*name", fields); + ImmutableSet availableFields = + ImmutableSet.of("firstname", "lastname", "age", "address", "fullname"); + + List nameFields = WildcardRenameUtils.matchFieldNames("*name", availableFields); assertEquals(List.of("firstname", "lastname", "fullname"), nameFields); - List firstFields = WildcardRenameUtils.matchFieldNames("first*", fields); + List firstFields = WildcardRenameUtils.matchFieldNames("first*", availableFields); assertEquals(List.of("firstname"), firstFields); - - // Test full wildcard - matches all fields - List allFields = WildcardRenameUtils.matchFieldNames("*", fields); + List allFields = WildcardRenameUtils.matchFieldNames("*", availableFields); assertEquals(List.of("firstname", "lastname", "age", "address", "fullname"), allFields); + } - // Test no matches - List noMatch = WildcardRenameUtils.matchFieldNames("*xyz", fields); + @Test + void testMatchFieldNamesNoMatches() { + ImmutableSet availableFields = + ImmutableSet.of("firstname", "lastname", "age", "address", "fullname"); + List noMatch = WildcardRenameUtils.matchFieldNames("*xyz", availableFields); assertTrue(noMatch.isEmpty()); + } - // Test exact match (no wildcards) - List exactMatch = WildcardRenameUtils.matchFieldNames("age", fields); + @Test + void testMatchFieldNamesNoWildcards() { + ImmutableSet availableFields = + ImmutableSet.of("firstname", "lastname", "age", "address", "fullname"); + List exactMatch = WildcardRenameUtils.matchFieldNames("age", availableFields); assertEquals(List.of("age"), exactMatch); - List exactNoMatch = WildcardRenameUtils.matchFieldNames("xyz", fields); + List exactNoMatch = WildcardRenameUtils.matchFieldNames("xyz", availableFields); assertTrue(exactNoMatch.isEmpty()); } @@ -84,8 +89,10 @@ void testApplyWildcardTransformation() { assertEquals( "prefixfirst", WildcardRenameUtils.applyWildcardTransformation("*name", "prefix*", "firstname")); + } - // Test full wildcard transformations + @Test + void testFullWildcardTransformation() { assertEquals( "firstname", WildcardRenameUtils.applyWildcardTransformation("*", "*", "firstname")); assertEquals( @@ -93,16 +100,17 @@ void testApplyWildcardTransformation() { WildcardRenameUtils.applyWildcardTransformation("*", "new_*", "firstname")); assertEquals( "first", WildcardRenameUtils.applyWildcardTransformation("*name", "*", "firstname")); + } - // Test partial match + @Test + void testPartialMatchWildcardTransformation() { assertEquals( "FiRsTname", WildcardRenameUtils.applyWildcardTransformation("f*r*tname", "F*R*Tname", "firstname")); } @Test - void testApplyWildcardTransformationErrors() { - // Test pattern mismatch - field doesn't match source pattern + void testApplyWildcardTransformationPatternMismatch() { assertThrows( IllegalArgumentException.class, () -> WildcardRenameUtils.applyWildcardTransformation("*name", "*NAME", "age")); @@ -110,20 +118,80 @@ void testApplyWildcardTransformationErrors() { @Test void testValidatePatternCompatibility() { - // Valid patterns assertTrue(WildcardRenameUtils.validatePatternCompatibility("*name", "*NAME")); assertTrue(WildcardRenameUtils.validatePatternCompatibility("*_*", "*_*")); assertTrue(WildcardRenameUtils.validatePatternCompatibility("prefix*suffix", "PREFIX*SUFFIX")); assertTrue(WildcardRenameUtils.validatePatternCompatibility("name", "NAME")); + } - // Valid full wildcard patterns + @Test + void testValidatePatternCompatibilityFullWildcard() { assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "*")); assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "new_*")); assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "*_old")); assertTrue(WildcardRenameUtils.validatePatternCompatibility("old_*", "*")); + } - // Invalid patterns - mismatched wildcard counts + @Test + void testValidatePatternCompatibilityInvalidPattern() { assertFalse(WildcardRenameUtils.validatePatternCompatibility("*name", "*_*")); assertFalse(WildcardRenameUtils.validatePatternCompatibility("*_*", "*")); } + + @Test + void testFullWildcardPatternTransformation() { + assertEquals( + "firstname", WildcardRenameUtils.applyWildcardTransformation("*", "*", "firstname")); + assertEquals( + "firstname", WildcardRenameUtils.applyWildcardTransformation("**", "**", "firstname")); + assertEquals( + "firstname", WildcardRenameUtils.applyWildcardTransformation("***", "***", "firstname")); + } + + @Test + void testSingleWildcardFullPatternTransformation() { + assertEquals( + "firstname_suffix", + WildcardRenameUtils.applyWildcardTransformation("*", "*_suffix", "firstname")); + assertEquals( + "prefix_firstname", + WildcardRenameUtils.applyWildcardTransformation("*", "prefix_*", "firstname")); + assertEquals( + "prefix_firstname_suffix", + WildcardRenameUtils.applyWildcardTransformation("*", "prefix_*_suffix", "firstname")); + } + + @Test + void testMultipleWildcardSourcePatternError() { + assertThrows( + IllegalArgumentException.class, + () -> WildcardRenameUtils.applyWildcardTransformation("**", "**_suffix", "firstname")); + assertThrows( + IllegalArgumentException.class, + () -> WildcardRenameUtils.applyWildcardTransformation("***", "prefix_***", "firstname")); + } + + @Test + void testValidatePatternCompatibilityMultipleWildcards() { + assertTrue(WildcardRenameUtils.validatePatternCompatibility("**", "**")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("***", "***")); + assertFalse(WildcardRenameUtils.validatePatternCompatibility("**", "*")); + assertFalse(WildcardRenameUtils.validatePatternCompatibility("*", "**")); + } + + @Test + void testConsecutiveWildcardsError() { + assertThrows( + IllegalArgumentException.class, + () -> WildcardRenameUtils.applyWildcardTransformation("*n**me", "*N**ME", "firstname")); + assertThrows( + IllegalArgumentException.class, + () -> + WildcardRenameUtils.applyWildcardTransformation("**name", "**something", "firstname")); + assertThrows( + IllegalArgumentException.class, + () -> + WildcardRenameUtils.applyWildcardTransformation( + "**name**", "**something**", "firstname")); + } } diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index 9c171f697ed..f5f129ca8f4 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -82,7 +82,7 @@ The example shows renaming multiple fields using wildcard patterns. (Requires Ca PPL query:: - PPL> source=accounts | rename *name as *_name | fields first_name, last_name; + os> source=accounts | rename *name as *_name | fields first_name, last_name; fetched rows / total rows = 4/4 +------------+-----------+ | first_name | last_name | @@ -101,7 +101,7 @@ The example shows renaming multiple fields using multiple wildcard patterns. (Re PPL query:: - PPL> source=accounts | rename *name as *_name, *_number as *number | fields first_name, last_name, accountnumber; + os> source=accounts | rename *name as *_name, *_number as *number | fields first_name, last_name, accountnumber; fetched rows / total rows = 4/4 +------------+-----------+---------------+ | first_name | last_name | accountnumber | @@ -120,7 +120,7 @@ The example shows renaming an existing field to an existing field. The target fi PPL query:: - PPL> source=accounts | rename firstname as age | fields age; + os> source=accounts | rename firstname as age | fields age; fetched rows / total rows = 4/4 +---------+ | age | diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index b746a66d498..0180b70e24a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -40,12 +40,7 @@ public void testRename() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); - verifyDataRows( - result, - rows("Jake", "USA", "California", 4, 2023, 70), - rows("Hello", "USA", "New York", 4, 2023, 30), - rows("John", "Canada", "Ontario", 4, 2023, 25), - rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + verifyStandardDataRows(result); } @Test @@ -85,12 +80,7 @@ public void testRenameToMetaField() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); - verifyDataRows( - result, - rows("Jake", "USA", "California", 4, 2023, 70), - rows("Hello", "USA", "New York", 4, 2023, 30), - rows("John", "Canada", "Ontario", 4, 2023, 25), - rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + verifyStandardDataRows(result); } @Test @@ -169,12 +159,7 @@ public void testRenameWildcardFields() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); - verifyDataRows( - result, - rows("Jake", "USA", "California", 4, 2023, 70), - rows("Hello", "USA", "New York", 4, 2023, 30), - rows("John", "Canada", "Ontario", 4, 2023, 25), - rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + verifyStandardDataRows(result); } @Test @@ -189,12 +174,7 @@ public void testRenameMultipleWildcardFields() throws IOException { schema("couNTry", "string"), schema("year", "int"), schema("moNTh", "int")); - verifyDataRows( - result, - rows("Jake", "USA", "California", 4, 2023, 70), - rows("Hello", "USA", "New York", 4, 2023, 30), - rows("John", "Canada", "Ontario", 4, 2023, 25), - rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + verifyStandardDataRows(result); } @Test @@ -209,12 +189,7 @@ public void testRenameWildcardPrefix() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); - verifyDataRows( - result, - rows("Jake", "USA", "California", 4, 2023, 70), - rows("Hello", "USA", "New York", 4, 2023, 30), - rows("John", "Canada", "Ontario", 4, 2023, 25), - rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + verifyStandardDataRows(result); } @Test @@ -240,12 +215,7 @@ public void testRenameMultipleWildcards() throws IOException { schema("country", "string"), schema("year", "int"), schema("MoNtH", "int")); - verifyDataRows( - result, - rows("Jake", "USA", "California", 4, 2023, 70), - rows("Hello", "USA", "New York", 4, 2023, 30), - rows("John", "Canada", "Ontario", 4, 2023, 25), - rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + verifyStandardDataRows(result); } @Test @@ -315,12 +285,7 @@ public void testRenamingNonExistentField() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); - verifyDataRows( - result, - rows("Jake", "USA", "California", 4, 2023, 70), - rows("Hello", "USA", "New York", 4, 2023, 30), - rows("John", "Canada", "Ontario", 4, 2023, 25), - rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + verifyStandardDataRows(result); } @Test @@ -365,6 +330,46 @@ public void testRenameSameField() throws IOException { schema("country", "string"), schema("year", "int"), schema("month", "int")); + verifyStandardDataRows(result); + } + + @Test + public void testMultipleRenameWithoutComma() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | rename name as user_name age as user_age country as location", + TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("user_name", "string"), + schema("user_age", "int"), + schema("state", "string"), + schema("location", "string"), + schema("year", "int"), + schema("month", "int")); + verifyStandardDataRows(result); + } + + @Test + public void testRenameMixedCommaAndSpace() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | rename name as user_name, age as user_age country as location", + TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("user_name", "string"), + schema("user_age", "int"), + schema("state", "string"), + schema("location", "string"), + schema("year", "int"), + schema("month", "int")); + verifyStandardDataRows(result); + } + + private void verifyStandardDataRows(JSONObject result) { verifyDataRows( result, rows("Jake", "USA", "California", 4, 2023, 70), diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java index 8fcd756edac..7a87df023a8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java @@ -266,4 +266,30 @@ public void testCrossClusterRegexWithNegation() throws IOException { rows("Amber JOHnny"), rows("Nanette")); } + + @Test + public void testCrossClusterRenameWildcardPattern() throws IOException { + JSONObject result = + executeQuery( + String.format("search source=%s | rename *ame as *AME", TEST_INDEX_DOG_REMOTE)); + verifyColumn(result, columnName("dog_nAME"), columnName("holdersNAME"), columnName("age")); + verifySchema( + result, + schema("dog_nAME", "string"), + schema("holdersNAME", "string"), + schema("age", "bigint")); + } + + @Test + public void testCrossClusterRenameFullWildcard() throws IOException { + JSONObject result = + executeQuery(String.format("search source=%s | rename * as old_*", TEST_INDEX_DOG_REMOTE)); + verifyColumn( + result, columnName("old_dog_name"), columnName("old_holdersName"), columnName("old_age")); + verifySchema( + result, + schema("old_dog_name", "string"), + schema("old_holdersName", "string"), + schema("old_age", "bigint")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java index e8ab7965a66..af1c08e8611 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CrossClusterSearchIT.java @@ -168,7 +168,7 @@ public void testMatchAllCrossClusterDescribeAllFields() throws IOException { columnName("IS_AUTOINCREMENT"), columnName("IS_GENERATEDCOLUMN")); } - + @Test public void testCrossClusterSortWithCount() throws IOException { JSONObject result = diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 9cb93826f7e..44404af4ecf 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -426,7 +426,7 @@ hintPair ; renameClasue - : orignalField = wcFieldExpression AS renamedField = wcFieldExpression + : orignalField = renameFieldExpression AS renamedField = renameFieldExpression ; byClause @@ -644,6 +644,11 @@ selectFieldExpression | STAR ; +renameFieldExpression + : wcQualifiedName + | STAR + ; + // functions evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS @@ -1212,7 +1217,6 @@ tableIdent wildcard : ident (MODULE ident)* (MODULE)? - | STAR | SINGLE_QUOTE wildcard SINGLE_QUOTE | DOUBLE_QUOTE wildcard DOUBLE_QUOTE | BACKTICK wildcard BACKTICK diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 0047161de96..36f8d4f58c5 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -32,6 +32,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalOrContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalXorContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.MultiFieldRelevanceFunctionContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RenameFieldExpressionContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SingleFieldRelevanceFunctionContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SortFieldContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SpanClauseContext; @@ -269,6 +270,14 @@ public UnresolvedExpression visitSelectFieldExpression( return new Field((QualifiedName) visit(ctx.wcQualifiedName())); } + @Override + public UnresolvedExpression visitRenameFieldExpression(RenameFieldExpressionContext ctx) { + if (ctx.STAR() != null) { + return new Field(QualifiedName.of("*")); + } + return new Field((QualifiedName) visit(ctx.wcQualifiedName())); + } + @Override public UnresolvedExpression visitSortField(SortFieldContext ctx) { From f8f2a6ac1cc147b8916c93c79425d79d9532fde8 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Tue, 2 Sep 2025 17:01:42 -0700 Subject: [PATCH 21/24] add back import Signed-off-by: Ritvi Bhatt --- core/src/main/java/org/opensearch/sql/analysis/Analyzer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 8c8fbd5404b..719818e4c78 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.tuple.ImmutablePair; From 6f4d4f245d9f5254d388574695b52acd6b4b2d51 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Tue, 2 Sep 2025 21:36:01 -0700 Subject: [PATCH 22/24] fix doc Signed-off-by: Ritvi Bhatt --- docs/category.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/category.json b/docs/category.json index a1c0667da14..a39acba05ee 100644 --- a/docs/category.json +++ b/docs/category.json @@ -20,7 +20,6 @@ "user/ppl/cmd/parse.rst", "user/ppl/cmd/patterns.rst", "user/ppl/cmd/rare.rst", - "user/ppl/cmd/rename.rst", "user/ppl/cmd/search.rst", "user/ppl/cmd/sort.rst", "user/ppl/cmd/syntax.rst", @@ -59,6 +58,7 @@ "user/ppl/cmd/stats.rst", "user/ppl/cmd/fields.rst", "user/ppl/cmd/regex.rst", - "user/ppl/cmd/stats.rst" + "user/ppl/cmd/stats.rst", + "user/ppl/cmd/rename.rst" ] } From 555c457f22fd560d6099b8ccc76e557205781735 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Thu, 4 Sep 2025 10:22:16 -0700 Subject: [PATCH 23/24] fix doc Signed-off-by: Ritvi Bhatt --- docs/category.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/category.json b/docs/category.json index 22255c9d57d..a69ea9945b9 100644 --- a/docs/category.json +++ b/docs/category.json @@ -59,7 +59,7 @@ "user/ppl/cmd/fields.rst", "user/ppl/cmd/regex.rst", "user/ppl/cmd/stats.rst", - "user/ppl/cmd/rename.rst" + "user/ppl/cmd/rename.rst", "user/ppl/cmd/timechart.rst" ] } From e8e71f13c43be43bdafbb42d4e17953281be6258 Mon Sep 17 00:00:00 2001 From: Ritvi Bhatt Date: Mon, 8 Sep 2025 09:45:34 -0700 Subject: [PATCH 24/24] fix formatting Signed-off-by: Ritvi Bhatt --- .../opensearch/sql/security/CalciteCrossClusterSearchIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java index 583725c29a6..1cbd019eca3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java @@ -292,7 +292,7 @@ public void testCrossClusterRenameFullWildcard() throws IOException { schema("old_holdersName", "string"), schema("old_age", "bigint")); } - + @Test public void testCrossClusterRexBasic() throws IOException { JSONObject result =