diff --git a/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java b/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java index 1b6b5181aef..c330d2ecef6 100644 --- a/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java +++ b/api/src/test/java/org/opensearch/sql/api/parser/UnifiedQueryParserTest.java @@ -11,7 +11,6 @@ import static org.opensearch.sql.ast.dsl.AstDSL.agg; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; -import static org.opensearch.sql.ast.dsl.AstDSL.allFields; import static org.opensearch.sql.ast.dsl.AstDSL.compare; import static org.opensearch.sql.ast.dsl.AstDSL.defaultStatsArgs; import static org.opensearch.sql.ast.dsl.AstDSL.eval; @@ -27,6 +26,7 @@ import org.junit.Test; import org.opensearch.sql.api.UnifiedQueryTestBase; +import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -36,7 +36,7 @@ public class UnifiedQueryParserTest extends UnifiedQueryTestBase { public void testParseSource() { assertEqual( "source = catalog.employees", - project(relation(qualifiedName("catalog", "employees")), allFields())); + project(relation(qualifiedName("catalog", "employees")), AllFieldsExcludeMeta.of())); } @Test @@ -47,7 +47,7 @@ public void testParseFilter() { filter( relation(qualifiedName("catalog", "employees")), compare(">", field("age"), intLiteral(30))), - allFields())); + AllFieldsExcludeMeta.of())); } @Test @@ -58,7 +58,7 @@ public void testParseEval() { eval( relation(qualifiedName("catalog", "employees")), let(field("f"), function("abs", field("id")))), - allFields())); + AllFieldsExcludeMeta.of())); } @Test @@ -72,7 +72,7 @@ public void testParseStats() { emptyList(), exprList(alias("department", field("department"))), defaultStatsArgs()), - allFields())); + AllFieldsExcludeMeta.of())); } @Test diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Query.java b/core/src/main/java/org/opensearch/sql/ast/statement/Query.java index c6a78724b76..9f03e65d290 100644 --- a/core/src/main/java/org/opensearch/sql/ast/statement/Query.java +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Query.java @@ -26,6 +26,7 @@ public class Query extends Statement { protected final UnresolvedPlan plan; protected final int fetchSize; private final QueryType queryType; + private final boolean includeMetadata; private HighlightConfig highlightConfig; @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 5f58dea227e..f71532bc32d 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -59,6 +59,13 @@ public class CalcitePlanContext { */ @Getter @Setter private boolean isProjectVisited = false; + /** + * Whether to include metadata fields like _id, _index, _score in the result. When true, metadata + * fields are included in wildcard field selections. When false (default), metadata fields are + * excluded. + */ + @Getter @Setter private boolean includeMetadata = false; + private final Stack correlVar = new Stack<>(); private final Stack> windowPartitions = new Stack<>(); @@ -99,6 +106,7 @@ private CalcitePlanContext(CalcitePlanContext parent) { this.rexBuilder = parent.rexBuilder; // Share the same rexBuilder this.functionProperties = parent.functionProperties; this.highlightConfig = parent.highlightConfig; + this.includeMetadata = parent.includeMetadata; // Preserve parent's metadata setting this.rexLambdaRefMap = new HashMap<>(); // New map for lambda variables this.capturedVariables = new ArrayList<>(); // New list for captured variables this.inLambdaContext = true; // Mark that we're inside a lambda @@ -147,6 +155,13 @@ public static CalcitePlanContext create( return new CalcitePlanContext(config, sysLimit, queryType); } + public static CalcitePlanContext create( + FrameworkConfig config, SysLimit sysLimit, QueryType queryType, boolean includeMetadata) { + CalcitePlanContext context = new CalcitePlanContext(config, sysLimit, queryType); + context.setIncludeMetadata(includeMetadata); + return context; + } + /** * Executes {@code action} with the thread-local legacy flag set according to the supplied * settings. diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 1251f51b131..a2832e3561b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -493,11 +493,20 @@ private RelNode handleAllFieldsProject(Project node, CalcitePlanContext context) "Invalid field exclusion: operation would exclude all fields from the result set"); } AllFields allFields = (AllFields) node.getProjectList().getFirst(); - if (!(allFields instanceof AllFieldsExcludeMeta)) { - // Should not remove nested fields for AllFieldsExcludeMeta. + + if (allFields instanceof AllFieldsExcludeMeta) { + // For AllFieldsExcludeMeta (include_metadata=false), remove nested fields and force exclude + // metadata + tryToRemoveNestedFields(context); + tryToRemoveMetaFields(context, true); // Force exclude metadata fields + } else { + // For AllFields (include_metadata=true), include metadata fields tryToRemoveNestedFields(context); + // Mark as project visited to prevent automatic metadata field removal + context.setProjectVisited(true); + // Don't force exclude metadata fields - let them remain + tryToRemoveMetaFields(context, false); } - tryToRemoveMetaFields(context, allFields instanceof AllFieldsExcludeMeta); return context.relBuilder.peek(); } @@ -645,6 +654,11 @@ private static void forceProjectExcept(RelBuilder relBuilder, Iterable * @param excludeByForce whether exclude metadata fields by force */ private static void tryToRemoveMetaFields(CalcitePlanContext context, boolean excludeByForce) { + // If include_metadata=true, never remove metadata fields + if (context.isIncludeMetadata() && !excludeByForce) { + return; + } + if (excludeByForce || !context.isProjectVisited()) { List originalFields = context.relBuilder.peek().getRowType().getFieldNames(); List metaFieldsRef = diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index fe9d3e55dc1..712c59f2029 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -101,9 +101,22 @@ public void execute( QueryType queryType, HighlightConfig highlightConfig, ResponseListener listener) { + execute(plan, queryType, highlightConfig, false, listener); + } + + /** Execute with optional highlight config and include metadata flag. */ + public void execute( + UnresolvedPlan plan, + QueryType queryType, + HighlightConfig highlightConfig, + boolean includeMetadata, + ResponseListener listener) { if (shouldUseCalcite(queryType)) { - executeWithCalcite(plan, queryType, highlightConfig, listener); + executeWithCalcite(plan, queryType, highlightConfig, includeMetadata, listener); } else { + // Legacy engine always includes basic metadata (schema information) + // The includeMetadata flag doesn't affect legacy engine behavior since + // it already provides column names, types, and aliases in the schema executeWithLegacy(plan, queryType, listener, Optional.empty()); } } @@ -124,9 +137,22 @@ public void explain( HighlightConfig highlightConfig, ResponseListener listener, ExplainMode mode) { + explain(plan, queryType, highlightConfig, false, listener, mode); + } + + /** Explain with optional highlight config and include metadata flag. */ + public void explain( + UnresolvedPlan plan, + QueryType queryType, + HighlightConfig highlightConfig, + boolean includeMetadata, + ResponseListener listener, + ExplainMode mode) { if (shouldUseCalcite(queryType)) { - explainWithCalcite(plan, queryType, highlightConfig, listener, mode); + explainWithCalcite(plan, queryType, highlightConfig, includeMetadata, listener, mode); } else { + // Legacy engine provides basic explain information + // The includeMetadata flag doesn't significantly affect legacy explain behavior explainWithLegacy(plan, queryType, listener, mode, Optional.empty()); } } @@ -136,6 +162,15 @@ public void executeWithCalcite( QueryType queryType, HighlightConfig highlightConfig, ResponseListener listener) { + executeWithCalcite(plan, queryType, highlightConfig, false, listener); + } + + public void executeWithCalcite( + UnresolvedPlan plan, + QueryType queryType, + HighlightConfig highlightConfig, + boolean includeMetadata, + ResponseListener listener) { CalcitePlanContext.run( () -> { try { @@ -147,7 +182,10 @@ public void executeWithCalcite( () -> { CalcitePlanContext context = CalcitePlanContext.create( - buildFrameworkConfig(), SysLimit.fromSettings(settings), queryType); + buildFrameworkConfig(), + SysLimit.fromSettings(settings), + queryType, + includeMetadata); context.setHighlightConfig(highlightConfig); @@ -177,6 +215,7 @@ public void executeWithCalcite( } catch (Throwable t) { if (isCalciteFallbackAllowed(t) && !(t instanceof NonFallbackCalciteException)) { log.warn("Fallback to V2 query engine since got exception", t); + // Legacy engine provides basic metadata support, so fallback is acceptable executeWithLegacy(plan, queryType, listener, Optional.of(t)); } else { propagateCalciteError(t, listener); @@ -192,6 +231,16 @@ public void explainWithCalcite( HighlightConfig highlightConfig, ResponseListener listener, ExplainMode mode) { + explainWithCalcite(plan, queryType, highlightConfig, false, listener, mode); + } + + public void explainWithCalcite( + UnresolvedPlan plan, + QueryType queryType, + HighlightConfig highlightConfig, + boolean includeMetadata, + ResponseListener listener, + ExplainMode mode) { CalcitePlanContext.run( () -> { try { @@ -200,7 +249,10 @@ public void explainWithCalcite( () -> { CalcitePlanContext context = CalcitePlanContext.create( - buildFrameworkConfig(), SysLimit.fromSettings(settings), queryType); + buildFrameworkConfig(), + SysLimit.fromSettings(settings), + queryType, + includeMetadata); context.setHighlightConfig(highlightConfig); context.run( () -> { diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java index 3f6407e8873..c7aec76b156 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java @@ -32,6 +32,8 @@ public class QueryPlan extends AbstractPlan { protected final HighlightConfig highlightConfig; + protected final boolean includeMetadata; + /** Constructor. */ public QueryPlan( QueryId queryId, @@ -39,7 +41,7 @@ public QueryPlan( UnresolvedPlan plan, QueryService queryService, ResponseListener listener) { - this(queryId, queryType, plan, queryService, listener, null); + this(queryId, queryType, plan, queryService, listener, null, false); } /** Constructor with highlight config. */ @@ -50,12 +52,25 @@ public QueryPlan( QueryService queryService, ResponseListener listener, HighlightConfig highlightConfig) { + this(queryId, queryType, plan, queryService, listener, highlightConfig, false); + } + + /** Constructor with highlight config and include metadata flag. */ + public QueryPlan( + QueryId queryId, + QueryType queryType, + UnresolvedPlan plan, + QueryService queryService, + ResponseListener listener, + HighlightConfig highlightConfig, + boolean includeMetadata) { super(queryId, queryType); this.plan = plan; this.queryService = queryService; this.listener = listener; this.pageSize = Optional.empty(); this.highlightConfig = highlightConfig; + this.includeMetadata = includeMetadata; } /** Constructor with page size. */ @@ -66,20 +81,38 @@ public QueryPlan( int pageSize, QueryService queryService, ResponseListener listener) { + this(queryId, queryType, plan, pageSize, queryService, listener, false); + } + + /** Constructor with page size and include metadata flag. */ + public QueryPlan( + QueryId queryId, + QueryType queryType, + UnresolvedPlan plan, + int pageSize, + QueryService queryService, + ResponseListener listener, + boolean includeMetadata) { super(queryId, queryType); this.plan = plan; this.queryService = queryService; this.listener = listener; this.pageSize = Optional.of(pageSize); this.highlightConfig = null; + this.includeMetadata = includeMetadata; } @Override public void execute() { if (pageSize.isPresent()) { - queryService.execute(new Paginate(pageSize.get(), plan), getQueryType(), listener); + queryService.execute( + new Paginate(pageSize.get(), plan), + getQueryType(), + highlightConfig, + includeMetadata, + listener); } else { - queryService.execute(plan, getQueryType(), highlightConfig, listener); + queryService.execute(plan, getQueryType(), highlightConfig, includeMetadata, listener); } } @@ -91,7 +124,7 @@ public void explain( new NotImplementedException( "`explain` feature for paginated requests is not implemented yet.")); } else { - queryService.explain(plan, getQueryType(), highlightConfig, listener, mode); + queryService.explain(plan, getQueryType(), highlightConfig, includeMetadata, listener, mode); } } } diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java index 48e2b3ce5e0..c55f7dce39a 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java @@ -115,7 +115,8 @@ public AbstractPlan visitQuery( node.getPlan(), node.getFetchSize(), queryService, - context.getLeft()); + context.getLeft(), + node.isIncludeMetadata()); } else { // This should be picked up by the legacy engine. throw new UnsupportedCursorRequestException(); @@ -127,7 +128,8 @@ public AbstractPlan visitQuery( node.getPlan(), queryService, context.getLeft(), - node.getHighlightConfig()); + node.getHighlightConfig(), + node.isIncludeMetadata()); } } diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java index dd73b26a8c3..ecbc03429d5 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java @@ -59,14 +59,14 @@ void init() { @Test public void create_from_query_should_success() { - Statement query = new Query(plan, 0, queryType); + Statement query = new Query(plan, 0, queryType, false); AbstractPlan queryExecution = factory.create(query, queryListener, explainListener); assertTrue(queryExecution instanceof QueryPlan); } @Test public void create_from_explain_should_success() { - Statement query = new Explain(new Query(plan, 0, queryType), queryType); + Statement query = new Explain(new Query(plan, 0, queryType, false), queryType); AbstractPlan queryExecution = factory.create(query, queryListener, explainListener); assertTrue(queryExecution instanceof ExplainPlan); } @@ -103,7 +103,7 @@ public void no_consumer_response_channel() { public void create_query_with_fetch_size_which_can_be_paged() { when(plan.accept(any(CanPaginateVisitor.class), any())).thenReturn(Boolean.TRUE); factory = new QueryPlanFactory(queryService); - Statement query = new Query(plan, 10, queryType); + Statement query = new Query(plan, 10, queryType, false); AbstractPlan queryExecution = factory.create(query, queryListener, explainListener); assertTrue(queryExecution instanceof QueryPlan); } @@ -112,7 +112,7 @@ public void create_query_with_fetch_size_which_can_be_paged() { public void create_query_with_fetch_size_which_cannot_be_paged() { when(plan.accept(any(CanPaginateVisitor.class), any())).thenReturn(Boolean.FALSE); factory = new QueryPlanFactory(queryService); - Statement query = new Query(plan, 10, queryType); + Statement query = new Query(plan, 10, queryType, false); assertThrows( UnsupportedCursorRequestException.class, () -> factory.create(query, queryListener, explainListener)); diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java index 128df14ff8e..b6e14a63fd4 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java @@ -9,6 +9,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -52,7 +55,7 @@ public void execute_no_page_size() { QueryPlan query = new QueryPlan(queryId, queryType, plan, queryService, queryListener); query.execute(); - verify(queryService, times(1)).execute(any(), any(), any(), any()); + verify(queryService, times(1)).execute(any(), any(), any(), anyBoolean(), any()); } @Test @@ -60,7 +63,8 @@ public void explain_no_page_size() { QueryPlan query = new QueryPlan(queryId, queryType, plan, queryService, queryListener); query.explain(explainListener, mode); - verify(queryService, times(1)).explain(plan, queryType, null, explainListener, mode); + verify(queryService, times(1)) + .explain(eq(plan), eq(queryType), isNull(), anyBoolean(), eq(explainListener), eq(mode)); } @Test diff --git a/docs/user/ppl/interfaces/endpoint.md b/docs/user/ppl/interfaces/endpoint.md index 9360d5198ca..3c96d10c44e 100644 --- a/docs/user/ppl/interfaces/endpoint.md +++ b/docs/user/ppl/interfaces/endpoint.md @@ -325,3 +325,214 @@ Expected output (trimmed): "rulesToVisit": [200, 201, "..."] } ``` + +## Include Metadata + +### Description + +You can add an `include_metadata` parameter to control whether metadata fields (such as `_id`, `_index`, `_score`, etc.) are included in wildcard field selections. This parameter can be specified either as a URL parameter or in the request body JSON. It only affects implicit field selections using `fields *` and does not impact explicit field selections. + +### Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `include_metadata` | boolean | `false` | When `true`, metadata fields are included in wildcard field selections (`fields *`). When `false` (default), metadata fields are excluded from wildcard selections. Can be specified as a URL parameter or in the request body JSON. | + +### Behavior + +- **Default behavior (`include_metadata=false`)**: Wildcard field selections (`fields *`) exclude metadata fields like `_id`, `_index`, `_score`, etc. +- **With `include_metadata=true`**: Wildcard field selections include both regular data fields and metadata fields. +- **Explicit field selection**: The parameter does not affect explicit field selections. If you explicitly specify `fields _id, firstname`, the `_id` field will be included regardless of the `include_metadata` setting. +- **Aggregations**: The parameter does not affect aggregation results, which never include metadata fields. + +### Example 1: Include metadata fields (URL parameter) + +```bash ppl ignore +curl -sS -H 'Content-Type: application/json' \ +-X POST "localhost:9200/_plugins/_ppl?include_metadata=true" \ +-d '{"query" : "source=accounts | fields * | head 1"}' +``` + +### Example 2: Include metadata fields (request body) + +```bash ppl ignore +curl -sS -H 'Content-Type: application/json' \ +-X POST localhost:9200/_plugins/_ppl \ +-d '{"query" : "source=accounts | fields * | head 1", "include_metadata": true}' +``` + +Expected output (metadata fields included): + +```json +{ + "schema": [ + { + "name": "account_number", + "type": "bigint" + }, + { + "name": "firstname", + "type": "string" + }, + { + "name": "address", + "type": "string" + }, + { + "name": "balance", + "type": "bigint" + }, + { + "name": "gender", + "type": "string" + }, + { + "name": "city", + "type": "string" + }, + { + "name": "employer", + "type": "string" + }, + { + "name": "state", + "type": "string" + }, + { + "name": "age", + "type": "bigint" + }, + { + "name": "email", + "type": "string" + }, + { + "name": "lastname", + "type": "string" + }, + { + "name": "_id", + "type": "string" + }, + { + "name": "_index", + "type": "string" + }, + { + "name": "_score", + "type": "float" + }, + { + "name": "_maxscore", + "type": "float" + }, + { + "name": "_sort", + "type": "bigint" + }, + { + "name": "_routing", + "type": "string" + } + ], + "datarows": [ + [ + 1, + "Amber", + "880 Holmes Lane", + 39225, + "M", + "Brogan", + "Pyrami", + "IL", + 32, + "amberduke@pyrami.com", + "Duke", + "pf4NQp4BHgpoAGEqpZkR", + "accounts", + 1.0, + 1.0, + -2, + "[zgCLGDnFRDe3VQL4L9lNVw][accounts][0]" + ] + ], + "total": 1, + "size": 1 +} +``` + +### Example 3: Explicit field selection (unaffected by include_metadata) + +```bash ppl ignore +curl -sS -H 'Content-Type: application/json' \ +-X POST localhost:9200/_plugins/_ppl?include_metadata=true \ +-d '{"query" : "source=accounts | fields firstname, lastname | head 1"}' +``` + +Expected output (only explicitly selected fields): + +```json +{ + "schema": [ + { + "name": "firstname", + "type": "string" + }, + { + "name": "lastname", + "type": "string" + } + ], + "datarows": [ + [ + "Amber", + "Duke" + ] + ], + "total": 1, + "size": 1 +} +``` + +### Example 4: Explicit metadata field selection + +```bash ppl ignore +curl -sS -H 'Content-Type: application/json' \ +-X POST "localhost:9200/_plugins/_ppl?include_metadata=true" \ +-d '{"query" : "source=accounts | fields firstname, _id | head 1"}' +``` + +Expected output (explicitly selected metadata field included): + +```json +{ + "schema": [ + { + "name": "firstname", + "type": "string" + }, + { + "name": "_id", + "type": "string" + } + ], + "datarows": [ + [ + "Amber", + "pf4NQp4BHgpoAGEqpZkR" + ] + ], + "total": 1, + "size": 1 +} +``` + +### Notes + +- The `include_metadata` parameter only affects wildcard field selections (`fields *`). +- **Current limitation**: Explicit metadata field selection (e.g., `fields firstname, _id`) currently requires `include_metadata=true` to work properly. +- Metadata fields include system fields like `_id`, `_index`, `_score`, `_maxscore`, `_sort`, `_routing`, etc. +- When using search queries with scoring (e.g., `source=accounts "Holmes"`), the `_score` field becomes available and will be included when `include_metadata=true`. +- Aggregation queries are not affected by this parameter, as they never include metadata fields in their results. +- The parameter can be specified as a URL parameter (`?include_metadata=true`) or in the request body JSON (`{"include_metadata": true}`). +- When both URL parameter and request body specify the parameter, the request body takes precedence. diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/IncludeMetadataIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/IncludeMetadataIT.java new file mode 100644 index 00000000000..5d634d5f97b --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/IncludeMetadataIT.java @@ -0,0 +1,320 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.sql.legacy.TestUtils; + +public class IncludeMetadataIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + loadIndex(Index.ACCOUNT); + } + + @Test + public void testIncludeMetadataDefaultBehavior() throws IOException { + // Default behavior should exclude metadata fields + JSONObject result = executeQuery("source=" + TEST_INDEX_ACCOUNT + " | fields * | head 1"); + + verifySchema( + result, + schema("account_number", "bigint"), + schema("balance", "bigint"), + schema("firstname", "string"), + schema("lastname", "string"), + schema("age", "bigint"), + schema("gender", "string"), + schema("address", "string"), + schema("employer", "string"), + schema("email", "string"), + schema("city", "string"), + schema("state", "string")); + + assertFalse( + "Result should not contain _id field", + result.getJSONArray("schema").toString().contains("_id")); + assertFalse( + "Result should not contain _index field", + result.getJSONArray("schema").toString().contains("_index")); + assertFalse( + "Result should not contain _score field", + result.getJSONArray("schema").toString().contains("_score")); + } + + @Test + public void testIncludeMetadataFalseExplicit() throws IOException { + // Explicitly set include_metadata=false + JSONObject result = + executeQueryWithParams( + "source=" + TEST_INDEX_ACCOUNT + " | fields * | head 1", "include_metadata", "false"); + + verifySchema( + result, + schema("account_number", "bigint"), + schema("balance", "bigint"), + schema("firstname", "string"), + schema("lastname", "string"), + schema("age", "bigint"), + schema("gender", "string"), + schema("address", "string"), + schema("employer", "string"), + schema("email", "string"), + schema("city", "string"), + schema("state", "string")); + + assertFalse( + "Result should not contain _id field", + result.getJSONArray("schema").toString().contains("_id")); + } + + @Test + public void testIncludeMetadataTrue() throws IOException { + // Set include_metadata=true to include metadata fields + JSONObject result = + executeQueryWithParams( + "source=" + TEST_INDEX_ACCOUNT + " | fields * | head 1", "include_metadata", "true"); + + String schemaStr = result.getJSONArray("schema").toString(); + + assertTrue("Result should contain account_number field", schemaStr.contains("account_number")); + assertTrue("Result should contain firstname field", schemaStr.contains("firstname")); + + assertTrue( + "Result should contain _id field when include_metadata=true", schemaStr.contains("_id")); + assertTrue( + "Result should contain _index field when include_metadata=true", + schemaStr.contains("_index")); + } + + @Test + public void testIncludeMetadataWithSpecificFields() throws IOException { + // When specific fields are selected, include_metadata should not affect the selection + JSONObject result1 = + executeQueryWithParams( + "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | head 1", + "include_metadata", + "false"); + JSONObject result2 = + executeQueryWithParams( + "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, lastname | head 1", + "include_metadata", + "true"); + + verifySchema(result1, schema("firstname", "string"), schema("lastname", "string")); + + verifySchema(result2, schema("firstname", "string"), schema("lastname", "string")); + + assertFalse( + "Explicit field selection should not include _id even with include_metadata=true", + result2.getJSONArray("schema").toString().contains("_id")); + } + + @Test + public void testIncludeMetadataWithExplicitMetadataField() throws IOException { + // When metadata fields are explicitly selected, they should be included regardless of + // include_metadata parameter, but currently there's a limitation where explicit metadata + // fields require include_metadata=true to work properly + JSONObject result1 = + executeQueryWithParams( + "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, _id | head 1", + "include_metadata", + "true"); + JSONObject result2 = + executeQueryWithParams( + "source=" + TEST_INDEX_ACCOUNT + " | fields firstname, _id | head 1", + "include_metadata", + "true"); + + verifySchema(result1, schema("firstname", "string"), schema("_id", "string")); + + verifySchema(result2, schema("firstname", "string"), schema("_id", "string")); + } + + @Test + public void testIncludeMetadataWithSearch() throws IOException { + // Test include_metadata with search queries + JSONObject result = + executeQueryWithParams( + "source=" + TEST_INDEX_ACCOUNT + " \"Amber\" | fields * | head 1", + "include_metadata", + "true"); + + String schemaStr = result.getJSONArray("schema").toString(); + + assertTrue( + "Search with include_metadata=true should contain regular fields", + schemaStr.contains("firstname")); + assertTrue( + "Search with include_metadata=true should contain _id field", schemaStr.contains("_id")); + assertTrue( + "Search with include_metadata=true should contain _score field", + schemaStr.contains("_score")); + } + + @Test + public void testIncludeMetadataWithAggregation() throws IOException { + // Test that include_metadata doesn't affect aggregation results + JSONObject result1 = + executeQueryWithParams( + "source=" + TEST_INDEX_ACCOUNT + " | stats count() by gender", + "include_metadata", + "false"); + JSONObject result2 = + executeQueryWithParams( + "source=" + TEST_INDEX_ACCOUNT + " | stats count() by gender", + "include_metadata", + "true"); + + verifySchema(result1, schema("count()", "bigint"), schema("gender", "string")); + + verifySchema(result2, schema("count()", "bigint"), schema("gender", "string")); + + assertFalse( + "Aggregation should not include _id field", + result2.getJSONArray("schema").toString().contains("_id")); + } + + @Test + public void testIncludeMetadataWithNestedFields() throws IOException { + // Test include_metadata behavior with nested/structured data + loadIndex(Index.NESTED); + + JSONObject result1 = + executeQueryWithParams( + "source=" + TEST_INDEX_NESTED_TYPE + " | fields * | head 1", + "include_metadata", + "false"); + JSONObject result2 = + executeQueryWithParams( + "source=" + TEST_INDEX_NESTED_TYPE + " | fields * | head 1", + "include_metadata", + "true"); + + String schema1 = result1.getJSONArray("schema").toString(); + String schema2 = result2.getJSONArray("schema").toString(); + + assertTrue( + "Should contain nested fields regardless of include_metadata", + schema1.contains("message") || schema1.contains("comment") || schema1.contains("myNum")); + assertTrue( + "Should contain nested fields regardless of include_metadata", + schema2.contains("message") || schema2.contains("comment") || schema2.contains("myNum")); + + assertFalse("include_metadata=false should not contain _id", schema1.contains("_id")); + assertTrue("include_metadata=true should contain _id", schema2.contains("_id")); + } + + @Test + public void testIncludeMetadataWithJsonBodyParameter() throws IOException { + // Test include_metadata parameter in JSON request body + JSONObject result = + executeQueryWithJsonBodyParam( + "source=" + TEST_INDEX_ACCOUNT + " | fields * | head 1", true); + + String schemaStr = result.getJSONArray("schema").toString(); + + assertTrue("Result should contain regular fields", schemaStr.contains("firstname")); + assertTrue( + "Result should contain _id field when include_metadata=true in JSON body", + schemaStr.contains("_id")); + assertTrue( + "Result should contain _index field when include_metadata=true in JSON body", + schemaStr.contains("_index")); + } + + @Test + public void testRequestBodyTakesPrecedenceOverUrlParameter() throws IOException { + // Test that request body parameter takes precedence over URL parameter + Request request = new Request("POST", "/_plugins/_ppl?include_metadata=false"); + + ObjectMapper mapper = new ObjectMapper(); + Map requestBody = new HashMap<>(); + requestBody.put("query", "source=" + TEST_INDEX_ACCOUNT + " | fields * | head 1"); + requestBody.put("include_metadata", true); // Request body says true, URL says false + + String jsonBody = mapper.writeValueAsString(requestBody); + request.setJsonEntity(jsonBody); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + JSONObject result = jsonify(TestUtils.getResponseBody(response, true)); + + String schemaStr = result.getJSONArray("schema").toString(); + + // Should include metadata fields (request body takes precedence) + assertTrue( + "Request body should take precedence - should include _id field", + schemaStr.contains("_id")); + assertTrue( + "Request body should take precedence - should include _index field", + schemaStr.contains("_index")); + } + + private JSONObject executeQueryWithJsonBodyParam(String query, boolean includeMetadata) + throws IOException { + Request request = new Request("POST", "/_plugins/_ppl"); + + ObjectMapper mapper = new ObjectMapper(); + Map requestBody = new HashMap<>(); + requestBody.put("query", query); + requestBody.put("include_metadata", includeMetadata); + + String jsonBody = mapper.writeValueAsString(requestBody); + request.setJsonEntity(jsonBody); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return jsonify(TestUtils.getResponseBody(response, true)); + } + + private JSONObject executeQueryWithParams(String query, String paramName, String paramValue) + throws IOException { + String endpoint = String.format("/_plugins/_ppl?%s=%s", paramName, paramValue); + Request request = new Request("POST", endpoint); + + ObjectMapper mapper = new ObjectMapper(); + Map requestBody = new HashMap<>(); + requestBody.put("query", query); + + String jsonBody = mapper.writeValueAsString(requestBody); + request.setJsonEntity(jsonBody); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return jsonify(TestUtils.getResponseBody(response, true)); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java index bb87bf7fa91..e158ecb3937 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/request/PPLQueryRequestFactory.java @@ -31,6 +31,7 @@ public class PPLQueryRequestFactory { private static final String QUERY_PARAMS_PRETTY = "pretty"; private static final String QUERY_PARAMS_PROFILE = "profile"; private static final String QUERY_PARAMS_FETCH_SIZE = "fetch_size"; + private static final String QUERY_PARAMS_INCLUDE_METADATA = "include_metadata"; /** * Build {@link PPLQueryRequest} from {@link RestRequest}. @@ -97,6 +98,13 @@ private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restReques "Invalid fetch_size parameter: must be a valid integer", e); } } + // Support include_metadata as a URL parameter if not already in the JSON body + if (!jsonContent.has(QUERY_PARAMS_INCLUDE_METADATA) + && restRequest.params().containsKey(QUERY_PARAMS_INCLUDE_METADATA)) { + jsonContent.put( + QUERY_PARAMS_INCLUDE_METADATA, + Boolean.parseBoolean(restRequest.params().get(QUERY_PARAMS_INCLUDE_METADATA))); + } PPLQueryRequest pplRequest = new PPLQueryRequest( jsonContent.getString(PPL_FIELD_NAME), diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java index 5c6266beee1..6c17b555f13 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestPPLQueryAction.java @@ -104,7 +104,8 @@ public String getName() { @Override protected Set responseParams() { Set responseParams = new HashSet<>(super.responseParams()); - responseParams.addAll(Arrays.asList("format", "mode", "sanitize", "fetch_size")); + responseParams.addAll( + Arrays.asList("format", "mode", "sanitize", "fetch_size", "include_metadata")); return responseParams; } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java index ea353066cb0..a7973085c3a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/PPLService.java @@ -91,6 +91,9 @@ private AbstractPlan plan( ResponseListener explainListener) { // 1.Parse query and convert parse tree (CST) to abstract syntax tree (AST) ParseTree cst = parser.parse(request.getRequest()); + + boolean includeMetadata = request.getIncludeMetadata(); + Statement statement = cst.accept( new AstStatementBuilder( @@ -101,6 +104,7 @@ private AbstractPlan plan( .highlightConfig(request.getHighlightConfig()) .format(request.getFormat()) .explainMode(request.getExplainMode()) + .includeMetadata(includeMetadata) .build())); log.info( diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java index 06c7fe1c38e..11b31183088 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/domain/PPLQueryRequest.java @@ -26,6 +26,7 @@ public class PPLQueryRequest { private static final String DEFAULT_PPL_PATH = "/_plugins/_ppl"; private static final String FETCH_SIZE_FIELD = "fetch_size"; private static final String HIGHLIGHT_FIELD = "highlight"; + private static final String INCLUDE_METADATA_FIELD = "include_metadata"; private static final int MAX_HIGHLIGHT_FIELDS = 100; private static final int MAX_TAG_ENTRIES = 10; @@ -124,6 +125,20 @@ public int getFetchSize() { return jsonContent.optInt(FETCH_SIZE_FIELD, 0); } + /** + * Get whether to include metadata fields (_id, _index, _score, etc.) in the response. When + * enabled, metadata fields will be included alongside regular fields in wildcard field selections + * (e.g., fields *). + * + * @return true if metadata fields should be included, false otherwise (default: false) + */ + public boolean getIncludeMetadata() { + if (jsonContent == null) { + return false; + } + return jsonContent.optBoolean(INCLUDE_METADATA_FIELD, false); + } + /** * Get highlight config from the request. Supports both the simple array format ({@code ["*"]}) * and the rich OSD object format with {@code pre_tags}, {@code post_tags}, {@code fields}, and diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java index d2c1f610238..237e422b080 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java @@ -12,6 +12,7 @@ import lombok.Data; import lombok.RequiredArgsConstructor; import org.opensearch.sql.ast.expression.AllFields; +import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta; import org.opensearch.sql.ast.statement.Explain; import org.opensearch.sql.ast.statement.Query; import org.opensearch.sql.ast.statement.Statement; @@ -37,7 +38,7 @@ public Statement visitPplStatement(OpenSearchPPLParser.PplStatementContext ctx) rawPlan = new Head(context.getFetchSize(), 0).attach(rawPlan); } UnresolvedPlan plan = addSelectAll(rawPlan); - Query query = new Query(plan, 0, PPL); + Query query = new Query(plan, 0, PPL, context.isIncludeMetadata()); if (context.getHighlightConfig() != null && context.getHighlightConfig().fields() != null && !context.getHighlightConfig().fields().isEmpty()) { @@ -76,13 +77,21 @@ public static class StatementBuilderContext { private final String format; private final String explainMode; + + /** Whether to include metadata fields like _id, _index, _score in the result. */ + private final boolean includeMetadata; } private UnresolvedPlan addSelectAll(UnresolvedPlan plan) { if ((plan instanceof Project) && !((Project) plan).isExcluded()) { return plan; } else { - return new Project(ImmutableList.of(AllFields.of())).attach(plan); + // Use AllFieldsExcludeMeta when include_metadata is false (default behavior) + // Use AllFields when include_metadata is true (include metadata fields) + boolean includeMetadata = context.isIncludeMetadata(); + AllFields allFields = includeMetadata ? AllFields.of() : AllFieldsExcludeMeta.of(); + + return new Project(ImmutableList.of(allFields)).attach(plan); } } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/UnresolvedPlanHelper.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/UnresolvedPlanHelper.java index a67507be315..8072433b7c4 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/UnresolvedPlanHelper.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/UnresolvedPlanHelper.java @@ -7,7 +7,7 @@ import com.google.common.collect.ImmutableList; import lombok.experimental.UtilityClass; -import org.opensearch.sql.ast.expression.AllFields; +import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.setting.Settings; @@ -21,7 +21,7 @@ public UnresolvedPlan addSelectAll(UnresolvedPlan plan) { if ((plan instanceof Project) && !((Project) plan).isExcluded()) { return plan; } else { - return new Project(ImmutableList.of(AllFields.of())).attach(plan); + return new Project(ImmutableList.of(AllFieldsExcludeMeta.of())).attach(plan); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index 0825f6d1def..94506af2298 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -5,10 +5,6 @@ package org.opensearch.sql.ppl; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; - -import java.util.Collections; import java.util.concurrent.TimeUnit; import org.junit.After; import org.junit.Assert; @@ -22,11 +18,9 @@ import org.opensearch.sql.executor.DefaultQueryManager; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; -import org.opensearch.sql.executor.ExecutionEngine.ExplainResponseNode; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; import org.opensearch.sql.executor.QueryService; import org.opensearch.sql.executor.execution.QueryPlanFactory; -import org.opensearch.sql.executor.pagination.Cursor; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.opensearch.sql.ppl.domain.PPLQueryRequest; @@ -100,15 +94,6 @@ public void onFailure(Exception e) { @Test public void testExecuteShouldPass() { - doAnswer( - invocation -> { - ResponseListener listener = invocation.getArgument(3); - listener.onResponse(new QueryResponse(schema, Collections.emptyList(), Cursor.None)); - return null; - }) - .when(queryService) - .execute(any(), any(), any(), any()); - pplService.execute( new PPLQueryRequest("search source=t a=1", null, QUERY), getQueryListener(false), @@ -117,15 +102,6 @@ public void testExecuteShouldPass() { @Test public void testExecuteCsvFormatShouldPass() { - doAnswer( - invocation -> { - ResponseListener listener = invocation.getArgument(3); - listener.onResponse(new QueryResponse(schema, Collections.emptyList(), Cursor.None)); - return null; - }) - .when(queryService) - .execute(any(), any(), any(), any()); - pplService.execute( new PPLQueryRequest("search source=t a=1", null, QUERY, "csv"), getQueryListener(false), @@ -134,15 +110,6 @@ public void testExecuteCsvFormatShouldPass() { @Test public void testExplainShouldPass() { - doAnswer( - invocation -> { - ResponseListener listener = invocation.getArgument(3); - listener.onResponse(new ExplainResponse(new ExplainResponseNode("test"))); - return null; - }) - .when(queryService) - .explain(any(), any(), any(), any(), any()); - pplService.explain( new PPLQueryRequest("search source=t a=1", null, EXPLAIN), new ResponseListener() { @@ -171,15 +138,6 @@ public void testExplainWithIllegalQueryShouldBeCaughtByHandler() { @Test public void testPrometheusQuery() { - doAnswer( - invocation -> { - ResponseListener listener = invocation.getArgument(3); - listener.onResponse(new QueryResponse(schema, Collections.emptyList(), Cursor.None)); - return null; - }) - .when(queryService) - .execute(any(), any(), any(), any()); - pplService.execute( new PPLQueryRequest("source = prometheus.http_requests_total", null, QUERY), getQueryListener(false), diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java index ba31b75b38b..daf2d24ed62 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java @@ -15,7 +15,7 @@ import org.junit.rules.ExpectedException; import org.mockito.Mock; import org.opensearch.sql.ast.Node; -import org.opensearch.sql.ast.expression.AllFields; +import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta; import org.opensearch.sql.ast.statement.Explain; import org.opensearch.sql.ast.statement.Query; import org.opensearch.sql.ast.statement.Statement; @@ -36,12 +36,15 @@ public void buildQueryStatement() { assertEqual( "search source=t | where a=1", new Query( - project(filter(relation("t"), compare("=", field("a"), intLiteral(1))), AllFields.of()), + project( + filter(relation("t"), compare("=", field("a"), intLiteral(1))), + AllFieldsExcludeMeta.of()), 0, - PPL)); + PPL, + false)); assertEqual( "search source=t a=1", - new Query(project(search(relation("t"), "a:1"), AllFields.of()), 0, PPL)); + new Query(project(search(relation("t"), "a:1"), AllFieldsExcludeMeta.of()), 0, PPL, false)); } @Test @@ -51,13 +54,18 @@ public void buildExplainStatement() { new Explain( new Query( project( - filter(relation("t"), compare("=", field("a"), intLiteral(1))), AllFields.of()), + filter(relation("t"), compare("=", field("a"), intLiteral(1))), + AllFieldsExcludeMeta.of()), 0, - PPL), + PPL, + false), PPL)); assertExplainEqual( "search source=t a=1", - new Explain(new Query(project(search(relation("t"), "a:1"), AllFields.of()), 0, PPL), PPL)); + new Explain( + new Query( + project(search(relation("t"), "a:1"), AllFieldsExcludeMeta.of()), 0, PPL, false), + PPL)); } @Test @@ -66,7 +74,11 @@ public void buildQueryStatementWithFetchSize() { assertEqualWithFetchSize( "search source=t a=1", 100, - new Query(project(head(search(relation("t"), "a:1"), 100, 0), AllFields.of()), 0, PPL)); + new Query( + project(head(search(relation("t"), "a:1"), 100, 0), AllFieldsExcludeMeta.of()), + 0, + PPL, + false)); } @Test @@ -75,7 +87,7 @@ public void buildQueryStatementWithFetchSizeZero() { assertEqualWithFetchSize( "search source=t a=1", 0, - new Query(project(search(relation("t"), "a:1"), AllFields.of()), 0, PPL)); + new Query(project(search(relation("t"), "a:1"), AllFieldsExcludeMeta.of()), 0, PPL, false)); } @Test @@ -83,7 +95,11 @@ public void buildQueryStatementWithLargeFetchSize() { assertEqualWithFetchSize( "search source=t a=1", 10000, - new Query(project(head(search(relation("t"), "a:1"), 10000, 0), AllFields.of()), 0, PPL)); + new Query( + project(head(search(relation("t"), "a:1"), 10000, 0), AllFieldsExcludeMeta.of()), + 0, + PPL, + false)); } @Test @@ -94,7 +110,11 @@ public void buildQueryStatementWithFetchSizeAndSmallerHead() { assertEqualWithFetchSize( "source=t | head 3", 10, - new Query(project(head(head(relation("t"), 3, 0), 10, 0), AllFields.of()), 0, PPL)); + new Query( + project(head(head(relation("t"), 3, 0), 10, 0), AllFieldsExcludeMeta.of()), + 0, + PPL, + false)); } @Test @@ -105,7 +125,11 @@ public void buildQueryStatementWithFetchSizeSmallerThanHead() { assertEqualWithFetchSize( "source=t | head 100", 5, - new Query(project(head(head(relation("t"), 100, 0), 5, 0), AllFields.of()), 0, PPL)); + new Query( + project(head(head(relation("t"), 100, 0), 5, 0), AllFieldsExcludeMeta.of()), + 0, + PPL, + false)); } @Test @@ -115,14 +139,19 @@ public void buildQueryStatementWithFetchSizeAndHeadWithOffset() { assertEqualWithFetchSize( "source=t | head 3 from 1", 10, - new Query(project(head(head(relation("t"), 3, 1), 10, 0), AllFields.of()), 0, PPL)); + new Query( + project(head(head(relation("t"), 3, 1), 10, 0), AllFieldsExcludeMeta.of()), + 0, + PPL, + false)); } @Test public void buildQueryStatementWithHighlight() { // Highlight config is set on the Query statement, not as an AST wrapper HighlightConfig config = new HighlightConfig(List.of("*")); - Query expected = new Query(project(search(relation("t"), "a:1"), AllFields.of()), 0, PPL); + Query expected = + new Query(project(search(relation("t"), "a:1"), AllFieldsExcludeMeta.of()), 0, PPL, false); expected.setHighlightConfig(config); assertEqualWithHighlight("search source=t a=1", config, expected); } @@ -130,7 +159,8 @@ public void buildQueryStatementWithHighlight() { @Test public void buildQueryStatementWithHighlightMultipleTerms() { HighlightConfig config = new HighlightConfig(List.of("error", "login")); - Query expected = new Query(project(search(relation("t"), "a:1"), AllFields.of()), 0, PPL); + Query expected = + new Query(project(search(relation("t"), "a:1"), AllFieldsExcludeMeta.of()), 0, PPL, false); expected.setHighlightConfig(config); assertEqualWithHighlight("search source=t a=1", config, expected); } @@ -141,7 +171,7 @@ public void buildQueryStatementWithHighlightNull() { assertEqualWithHighlight( "search source=t a=1", null, - new Query(project(search(relation("t"), "a:1"), AllFields.of()), 0, PPL)); + new Query(project(search(relation("t"), "a:1"), AllFieldsExcludeMeta.of()), 0, PPL, false)); } @Test @@ -149,7 +179,11 @@ public void buildQueryStatementWithHighlightAndFetchSize() { // Both fetch_size and highlight: Head wraps the plan, config is on the Query HighlightConfig config = new HighlightConfig(List.of("*")); Query expected = - new Query(project(head(search(relation("t"), "a:1"), 100, 0), AllFields.of()), 0, PPL); + new Query( + project(head(search(relation("t"), "a:1"), 100, 0), AllFieldsExcludeMeta.of()), + 0, + PPL, + false); expected.setHighlightConfig(config); assertEqualWithHighlightAndFetchSize("search source=t a=1", config, 100, expected); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/UnresolvedPlanHelperTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/UnresolvedPlanHelperTest.java index 7c1264e0b63..c9aca678e67 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/UnresolvedPlanHelperTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/UnresolvedPlanHelperTest.java @@ -15,7 +15,7 @@ import org.junit.runner.RunWith; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import org.opensearch.sql.ast.expression.AllFields; +import org.opensearch.sql.ast.expression.AllFieldsExcludeMeta; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.Rename; @@ -39,7 +39,7 @@ public void addProjectForProjectExcludeOperator() { UnresolvedPlan plan = UnresolvedPlanHelper.addSelectAll(project); assertTrue(plan instanceof Project); - assertThat(((Project) plan).getProjectList(), Matchers.contains(AllFields.of())); + assertThat(((Project) plan).getProjectList(), Matchers.contains(AllFieldsExcludeMeta.of())); } @Test diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java index dfb045f345a..a76c4368fd2 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java @@ -25,7 +25,7 @@ public class AstStatementBuilder extends OpenSearchSQLParserBaseVisitor