From c7b171d14d01569711ef969a5c3c7541df04bb9a Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Tue, 19 May 2026 16:08:29 -0700 Subject: [PATCH 1/2] Remove DatetimeOutputCastRule on the analytics-engine route (#5420) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the analytics-engine route, every datetime root column is wrapped in `CAST( AS VARCHAR)` by `DatetimeOutputCastRule`. A matching `DatetimeOutputCastRewriter` on the engine side then translates that cast into DataFusion's `to_char` extension. Whenever the rewriter's format string and the PPL formatter disagree (e.g. trailing `Z`, `T` separator), users see wire-format divergence — issue #5420. Drop the cast rule and let the engine return real datetime cells. The PPL response pipeline already handles datetime → string conversion natively at the formatter layer: - `AnalyticsExecutionEngine.convertRows` feeds engine cells (`LocalDateTime` / `LocalDate` / `LocalTime`) through `ExprValueUtils.fromObjectValue`, which produces `ExprTimestampValue` / `ExprDateValue` / `ExprTimeValue`. - Their `value()` methods render the documented PPL space-separated format (`yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]` etc.). The companion change in `opensearch-project/OpenSearch` removes the matching `DatetimeOutputCastRewriter` and its callsites. - Delete `DatetimeOutputCastRule`. - Drop the unused `UdtMapping.isDatetimeType` helper. - Rewrite the four cast-shape assertions in `DatetimeExtensionTest` to assert the post-removal RelNode (no `CAST(... VARCHAR)` wrapper, schema reports `DATE` / `TIME` / `TIMESTAMP`). Signed-off-by: Eric Wei --- .../api/spec/datetime/DatetimeExtension.java | 11 +--- .../spec/datetime/DatetimeOutputCastRule.java | 62 ------------------- .../datetime/DatetimeExtensionSqlTest.java | 23 +++---- .../spec/datetime/DatetimeExtensionTest.java | 48 +++++++------- .../remote/CalcitePlannerConcurrencyIT.java | 7 +-- 5 files changed, 38 insertions(+), 113 deletions(-) delete mode 100644 api/src/main/java/org/opensearch/sql/api/spec/datetime/DatetimeOutputCastRule.java diff --git a/api/src/main/java/org/opensearch/sql/api/spec/datetime/DatetimeExtension.java b/api/src/main/java/org/opensearch/sql/api/spec/datetime/DatetimeExtension.java index 1f0b0b820a..bd6a2c207c 100644 --- a/api/src/main/java/org/opensearch/sql/api/spec/datetime/DatetimeExtension.java +++ b/api/src/main/java/org/opensearch/sql/api/spec/datetime/DatetimeExtension.java @@ -17,13 +17,13 @@ import org.opensearch.sql.calcite.type.AbstractExprRelDataType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; -/** Datetime language extension that normalizes UDT types and casts output for wire-format. */ +/** Datetime language extension that normalizes datetime UDT types to standard Calcite types. */ public class DatetimeExtension implements LanguageExtension { @Override public List postAnalysisRules() { - // Fresh instances per plan() because RelHomogeneousShuttle inherits a stateful stack. - return List.of(new DatetimeUdtNormalizeRule(), new DatetimeOutputCastRule()); + // Fresh instance per plan() because RelHomogeneousShuttle inherits a stateful stack. + return List.of(new DatetimeUdtNormalizeRule()); } /** Maps datetime UDT types to their standard Calcite equivalents. */ @@ -45,10 +45,5 @@ static Optional fromUdtType(RelDataType type) { ExprUDT udt = e.getUdt(); return Arrays.stream(values()).filter(u -> u.udtType == udt).findFirst(); } - - /** Returns true if the given SqlTypeName is a standard datetime type. */ - static boolean isDatetimeType(SqlTypeName typeName) { - return Arrays.stream(values()).anyMatch(u -> u.stdType == typeName); - } } } diff --git a/api/src/main/java/org/opensearch/sql/api/spec/datetime/DatetimeOutputCastRule.java b/api/src/main/java/org/opensearch/sql/api/spec/datetime/DatetimeOutputCastRule.java deleted file mode 100644 index edc418928c..0000000000 --- a/api/src/main/java/org/opensearch/sql/api/spec/datetime/DatetimeOutputCastRule.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.api.spec.datetime; - -import static org.opensearch.sql.api.spec.datetime.DatetimeExtension.UdtMapping.isDatetimeType; - -import java.util.ArrayList; -import java.util.List; -import org.apache.calcite.rel.RelHomogeneousShuttle; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.type.SqlTypeName; - -/** - * Wraps the root output with CAST(datetime → VARCHAR) for PPL wire-format compatibility. - * - *

Not a singleton: {@link RelHomogeneousShuttle} inherits a stateful {@code stack} field from - * {@link org.apache.calcite.rel.RelShuttleImpl}, so a fresh instance must be used per plan(). - */ -class DatetimeOutputCastRule extends RelHomogeneousShuttle { - - @Override - public RelNode visit(RelNode other) { - List fields = other.getRowType().getFieldList(); - if (fields.stream().noneMatch(f -> isDatetimeType(f.getType().getSqlTypeName()))) { - return other; - } - - RexBuilder rexBuilder = other.getCluster().getRexBuilder(); - List projects = new ArrayList<>(fields.size()); - List names = new ArrayList<>(fields.size()); - - // Cast datetime fields to VARCHAR for output; pass through others unchanged - for (RelDataTypeField field : fields) { - RexNode newField = rexBuilder.makeInputRef(other, field.getIndex()); - RelDataType fieldType = field.getType(); - if (isDatetimeType(fieldType.getSqlTypeName())) { - projects.add(castToVarchar(rexBuilder, newField, fieldType)); - } else { - projects.add(newField); - } - names.add(field.getName()); - } - return LogicalProject.create(other, List.of(), projects, names); - } - - private static RexNode castToVarchar(RexBuilder rexBuilder, RexNode expr, RelDataType fieldType) { - RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory(); - RelDataType varcharType = - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.VARCHAR), fieldType.isNullable()); - return rexBuilder.makeCast(varcharType, expr); - } -} diff --git a/api/src/test/java/org/opensearch/sql/api/spec/datetime/DatetimeExtensionSqlTest.java b/api/src/test/java/org/opensearch/sql/api/spec/datetime/DatetimeExtensionSqlTest.java index 3cae8abd9b..dc1e94a489 100644 --- a/api/src/test/java/org/opensearch/sql/api/spec/datetime/DatetimeExtensionSqlTest.java +++ b/api/src/test/java/org/opensearch/sql/api/spec/datetime/DatetimeExtensionSqlTest.java @@ -20,8 +20,8 @@ import org.opensearch.sql.executor.QueryType; /** - * Tests that DatetimeExtension post-analysis rules (UDT normalization and output VARCHAR cast) - * apply correctly to the SQL V2 parser path through CalciteRelNodeVisitor. + * Tests that the DatetimeExtension UDT-normalization post-analysis rule applies correctly to the + * SQL V2 parser path through CalciteRelNodeVisitor. */ public class DatetimeExtensionSqlTest extends UnifiedQueryTestBase { @@ -47,7 +47,7 @@ protected Map getTableMap() { private Table createEventsTable() { return SimpleTable.builder() .col("id", INTEGER) - .col("name", VARCHAR) + .col("event_str", VARCHAR) .col("hire_date", DATE) .col("start_time", TIME) .col("created_at", TIMESTAMP) @@ -57,12 +57,11 @@ private Table createEventsTable() { } @Test - public void testAllStandardDatetimeTypesCastToVarchar() { + public void testStandardDatetimeTypesNotWrapped() { givenQuery("SELECT * FROM catalog.events") .assertPlan( """ - LogicalProject(id=[$0], name=[$1], hire_date=[CAST($2):VARCHAR NOT NULL], start_time=[CAST($3):VARCHAR NOT NULL], created_at=[CAST($4):VARCHAR NOT NULL]) - LogicalTableScan(table=[[catalog, events]]) + LogicalTableScan(table=[[catalog, events]]) """); } @@ -71,21 +70,19 @@ public void testFilterWithTimestampLiteral() { givenQuery("SELECT * FROM catalog.events WHERE created_at > '2024-01-01T00:00:00Z'") .assertPlan( """ - LogicalProject(id=[$0], name=[$1], hire_date=[CAST($2):VARCHAR NOT NULL], start_time=[CAST($3):VARCHAR NOT NULL], created_at=[CAST($4):VARCHAR NOT NULL]) - LogicalFilter(condition=[>($4, TIMESTAMP('2024-01-01T00:00:00Z':VARCHAR))]) - LogicalTableScan(table=[[catalog, events]]) + LogicalFilter(condition=[>($4, TIMESTAMP('2024-01-01T00:00:00Z':VARCHAR))]) + LogicalTableScan(table=[[catalog, events]]) """) .assertReturnType("TIMESTAMP", TIMESTAMP, 9); } @Test public void testComparisonWithDatetimeUdf() { - givenQuery("SELECT * FROM catalog.events WHERE created_at < DATE(name)") + givenQuery("SELECT * FROM catalog.events WHERE created_at < DATE(event_str)") .assertPlan( """ - LogicalProject(id=[$0], name=[$1], hire_date=[CAST($2):VARCHAR NOT NULL], start_time=[CAST($3):VARCHAR NOT NULL], created_at=[CAST($4):VARCHAR NOT NULL]) - LogicalFilter(condition=[<($4, TIMESTAMP(DATE($1)))]) - LogicalTableScan(table=[[catalog, events]]) + LogicalFilter(condition=[<($4, TIMESTAMP(DATE($1)))]) + LogicalTableScan(table=[[catalog, events]]) """) .assertReturnType("DATE", DATE) .assertReturnType("TIMESTAMP", TIMESTAMP, 9); diff --git a/api/src/test/java/org/opensearch/sql/api/spec/datetime/DatetimeExtensionTest.java b/api/src/test/java/org/opensearch/sql/api/spec/datetime/DatetimeExtensionTest.java index b0462f7370..b6f4f3562c 100644 --- a/api/src/test/java/org/opensearch/sql/api/spec/datetime/DatetimeExtensionTest.java +++ b/api/src/test/java/org/opensearch/sql/api/spec/datetime/DatetimeExtensionTest.java @@ -53,7 +53,7 @@ public void setUp() { private Table createEventsTable() { return SimpleTable.builder() .col("id", INTEGER) - .col("name", VARCHAR) + .col("event_str", VARCHAR) .col("hire_date", DATE) .col("start_time", TIME) .col("created_at", TIMESTAMP) @@ -63,18 +63,17 @@ private Table createEventsTable() { } @Test - public void testUdfResultNormalizedAndCastToVarchar() { + public void testUdfResultNormalized() { givenQuery( """ source = catalog.events \ - | eval d = DATE(name), t = TIME(name), ts = TIMESTAMP(name) \ + | eval d = DATE(event_str), t = TIME(event_str), ts = TIMESTAMP(event_str) \ | fields d, t, ts\ """) .assertPlan( """ - LogicalProject(d=[CAST($0):VARCHAR], t=[CAST($1):VARCHAR], ts=[CAST($2):VARCHAR]) - LogicalProject(d=[DATE($1)], t=[TIME($1)], ts=[TIMESTAMP($1)]) - LogicalTableScan(table=[[catalog, events]]) + LogicalProject(d=[DATE($1)], t=[TIME($1)], ts=[TIMESTAMP($1)]) + LogicalTableScan(table=[[catalog, events]]) """) .assertReturnType("DATE", DATE) .assertReturnType("TIME", TIME, 9) @@ -83,7 +82,9 @@ public void testUdfResultNormalizedAndCastToVarchar() { @Test public void testNestedUdfCallsNormalized() { - givenQuery("source = catalog.events | eval d = DATEDIFF(DATE(name), DATE(name)) | fields d") + givenQuery( + "source = catalog.events | eval d = DATEDIFF(DATE(event_str), DATE(event_str)) | fields" + + " d") .assertPlan( """ LogicalProject(d=[DATEDIFF(DATE($1), DATE($1))]) @@ -94,13 +95,12 @@ public void testNestedUdfCallsNormalized() { } @Test - public void testDateLiteralCastToVarchar() { + public void testDateLiteralNormalized() { givenQuery("source = catalog.events | eval d = DATE('2024-01-01') | fields d") .assertPlan( """ - LogicalProject(d=[CAST($0):VARCHAR]) - LogicalProject(d=[DATE('2024-01-01':VARCHAR)]) - LogicalTableScan(table=[[catalog, events]]) + LogicalProject(d=[DATE('2024-01-01':VARCHAR)]) + LogicalTableScan(table=[[catalog, events]]) """) .assertReturnType("DATE", DATE); } @@ -122,7 +122,7 @@ public void testFilterWithTimestampLiteral() { @Test public void testComparisonWithDatetimeUdf() { - givenQuery("source = catalog.events | where created_at < DATE(name) | fields id") + givenQuery("source = catalog.events | where created_at < DATE(event_str) | fields id") .assertPlan( """ LogicalProject(id=[$0]) @@ -134,22 +134,21 @@ public void testComparisonWithDatetimeUdf() { } @Test - public void testAllStandardDatetimeTypesCastToVarchar() { + public void testStandardDatetimeFieldsNotWrapped() { givenQuery("source = catalog.events | fields hire_date, start_time, created_at") .assertPlan( """ - LogicalProject(hire_date=[CAST($0):VARCHAR NOT NULL], start_time=[CAST($1):VARCHAR NOT NULL], created_at=[CAST($2):VARCHAR NOT NULL]) - LogicalProject(hire_date=[$2], start_time=[$3], created_at=[$4]) - LogicalTableScan(table=[[catalog, events]]) + LogicalProject(hire_date=[$2], start_time=[$3], created_at=[$4]) + LogicalTableScan(table=[[catalog, events]]) """); } @Test public void testNonDatetimeFieldsNotWrapped() { - givenQuery("source = catalog.events | fields id, name") + givenQuery("source = catalog.events | fields id, event_str") .assertPlan( """ - LogicalProject(id=[$0], name=[$1]) + LogicalProject(id=[$0], event_str=[$1]) LogicalTableScan(table=[[catalog, events]]) """); } @@ -168,7 +167,7 @@ public void testSequentialPlanCallsDoNotCorruptShuttleStack() { + " | stats count() as field_count, distinct_count(created_at) as distinct_count"); planner.plan( "source = catalog.events" - + " | eval ts = TIMESTAMP(name)" + + " | eval ts = TIMESTAMP(event_str)" + " | stats count() as field_count, distinct_count(ts) as distinct_count"); planner.plan( "source = catalog.events | where created_at > \"2024-01-01\" | fields hire_date"); @@ -176,19 +175,16 @@ public void testSequentialPlanCallsDoNotCorruptShuttleStack() { } @Test - public void testOutputCastCanCompileAndExecute() throws Exception { + public void testDatetimeFieldsPreserveStandardTypes() throws Exception { RelNode plan = planner.plan("source = catalog.events | fields hire_date, start_time, created_at"); try (PreparedStatement statement = compiler.compile(plan)) { ResultSet resultSet = statement.executeQuery(); verify(resultSet) .expectSchema( - col("hire_date", java.sql.Types.VARCHAR), - col("start_time", java.sql.Types.VARCHAR), - col("created_at", java.sql.Types.VARCHAR)) - .expectData( - row("2024-01-16", "12:00:00", "2024-01-15 08:00:00"), - row("2024-06-20", "14:00:00", "2024-06-20 00:00:00")); + col("hire_date", java.sql.Types.DATE), + col("start_time", java.sql.Types.TIME), + col("created_at", java.sql.Types.TIMESTAMP)); } } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePlannerConcurrencyIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePlannerConcurrencyIT.java index 375a0f1071..26509cb0f3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePlannerConcurrencyIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePlannerConcurrencyIT.java @@ -31,10 +31,9 @@ * *

The test methods here fire many queries through a thread pool to exercise concurrent {@code * plan()} invocations. New planner-level concurrency / state-isolation regressions belong in this - * class. The current cases cover {@code DatetimeExtension}'s {@code RelHomogeneousShuttle} - * subclasses ({@code DatetimeUdtNormalizeRule}, {@code DatetimeOutputCastRule}) which were - * previously returned as static {@code INSTANCE}s and caused the production failure that motivated - * this suite. + * class. The current cases cover {@code DatetimeExtension}'s {@code RelHomogeneousShuttle} subclass + * {@code DatetimeUdtNormalizeRule}, which was previously returned as a static {@code INSTANCE} and + * caused the production failure that motivated this suite. * *

Run via: * From 15e883ebb9e79551cfc0c3d19a15fcf722ac35a1 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Wed, 20 May 2026 15:49:27 -0700 Subject: [PATCH 2/2] Add CalciteAnalyticsDatetimeWireFormatIT regression net for #5420 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire-format regression coverage for sql#5420. With DatetimeOutputCastRule deleted (sql#5454) and DatetimeOutputCastRewriter deleted (opensearch#21748), datetime root columns must reach the user as PPL's documented `yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]` format with typed schema labels (`timestamp` / `date` / `time`, never `string`) on the analytics-engine route. The IT skips cleanly when `-Dtests.analytics.parquet_indices=true` is not set — Calcite-legacy was never affected by sql#5420 and asserting the same contract on it is duplicative noise. Coverage: - Wire-format round trip (typed schema + space-separator value) on TIMESTAMP / DATE / TIME root columns, plus eval-derived TIMESTAMP and `min(ts)` aggregation. - Datetime processing inside AE (parsing for WHERE comparison, scalar extract functions year/month/day/hour, ORDER BY). - Nanosecond precision preservation via `date_nanos`. - Aggregation beyond min(): max(ts), dc(ts). Each test asserts the query routes to AE (LogicalTableScan with lowercase `opensearch`) before checking wire format, so a future regression that silently routes to Calcite-legacy can't leave the contract green by accident. Signed-off-by: Eric Wei --- .../CalciteAnalyticsDatetimeWireFormatIT.java | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java new file mode 100644 index 0000000000..36dcf5697c --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAnalyticsDatetimeWireFormatIT.java @@ -0,0 +1,227 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.junit.Assume.assumeTrue; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.sql.legacy.TestUtils; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Regression net for sql#5420 on the analytics-engine route. Pins datetime wire format ({@code + * yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]}, typed schema labels) and asserts every query was served by AE — + * without the routing pin, a silent fallback to Calcite would leave the assertions green (Calcite + * already emits the documented format). Skipped on the legacy path. + */ +public class CalciteAnalyticsDatetimeWireFormatIT extends PPLIntegTestCase { + + private static final String INDEX = "wire_format_dt"; + + @Override + public void init() throws Exception { + super.init(); + assumeTrue( + "CalciteAnalyticsDatetimeWireFormatIT only meaningful with" + + " -Dtests.analytics.parquet_indices=true", + isAnalyticsParquetIndicesEnabled()); + enableCalcite(); + + if (!TestUtils.isIndexExist(client(), INDEX)) { + String mapping = + "{\"mappings\":{\"properties\":{" + + "\"ts\":{\"type\":\"date\",\"format\":\"yyyy-MM-dd HH:mm:ss\"}," + + "\"ts_nanos\":{\"type\":\"date_nanos\"}," + + "\"d\":{\"type\":\"date\",\"format\":\"yyyy-MM-dd\"}," + + "\"t\":{\"type\":\"date\",\"format\":\"HH:mm:ss\"}}}}"; + TestUtils.createIndexByRestClient(client(), INDEX, mapping); + + Request doc = new Request("PUT", "/" + INDEX + "/_doc/1?refresh=true"); + doc.setJsonEntity( + "{\"ts\":\"2024-03-15 10:30:00\"," + + "\"ts_nanos\":\"2024-03-15T10:30:00.123456789Z\"," + + "\"d\":\"2024-03-15\"," + + "\"t\":\"10:30:00\"}"); + client().performRequest(doc); + + Request doc2 = new Request("PUT", "/" + INDEX + "/_doc/2?refresh=true"); + doc2.setJsonEntity( + "{\"ts\":\"2024-03-16 23:59:59\"," + + "\"ts_nanos\":\"2024-03-16T23:59:59.999999999Z\"," + + "\"d\":\"2024-03-16\"," + + "\"t\":\"23:59:59\"}"); + client().performRequest(doc2); + } + } + + /** + * AE route: {@code LogicalTableScan} + lowercase {@code opensearch}. Calcite legacy uses {@code + * CalciteLogicalIndexScan}. + */ + private void assertRoutedToAnalyticsEngine(String query) throws IOException { + String explained = explainQueryToString(query); + Assert.assertTrue( + "Expected analytics-engine route (LogicalTableScan + lowercase 'opensearch'), got: " + + explained, + explained.contains("LogicalTableScan(table=[[opensearch,")); + Assert.assertFalse( + "Expected analytics-engine route, but query routed to Calcite legacy" + + " (CalciteLogicalIndexScan): " + + explained, + explained.contains("CalciteLogicalIndexScan")); + } + + /** TIMESTAMP root col: typed schema + space-separator value. */ + @Test + public void testTimestampRootColumnSpaceFormat() throws IOException { + String query = "source=" + INDEX + " | where ts = '2024-03-15 10:30:00' | fields ts"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifySchema(result, schema("ts", "timestamp")); + verifyDataRows(result, rows("2024-03-15 10:30:00")); + } + + /** + * DATE-mapped col: AE widens to TIMESTAMP at scan time; value must use space separator, not ISO + * {@code T}. + */ + @Test + public void testDateRootColumnYmdFormat() throws IOException { + String query = "source=" + INDEX + " | where d = '2024-03-15' | fields d"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifySchema(result, schema("d", "timestamp")); + verifyDataRows(result, rows("2024-03-15 00:00:00")); + } + + /** TIME-mapped col: AE widens to TIMESTAMP; value must use space separator, not ISO {@code T}. */ + @Test + public void testTimeRootColumnHmsFormat() throws IOException { + String query = "source=" + INDEX + " | sort t | head 1 | fields t"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifySchema(result, schema("t", "timestamp")); + Assert.assertFalse( + "Time-mapped column must not surface as ISO T-separator literal", + result.getJSONArray("datarows").getJSONArray(0).getString(0).contains("T")); + } + + /** Eval-derived TIMESTAMP follows the same wire-format contract as a root column. */ + @Test + public void testEvalDerivedTimestampSpaceFormat() throws IOException { + String query = + "source=" + INDEX + " | where ts = '2024-03-15 10:30:00' | eval x = ts | fields x"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifySchema(result, schema("x", "timestamp")); + verifyDataRows(result, rows("2024-03-15 10:30:00")); + } + + /** {@code min(ts)} returns a typed timestamp cell, not a stringified ISO-T literal. */ + @Test + public void testStatsMinTimestampSpaceFormat() throws IOException { + String query = "source=" + INDEX + " | stats min(ts) as min_ts"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifySchema(result, schema("min_ts", "timestamp")); + verifyDataRows(result, rows("2024-03-15 10:30:00")); + } + + /** + * AE parses indexed TIMESTAMP as a real timestamp for WHERE comparison (not lex string compare). + */ + @Test + public void testTimestampWhereComparisonFiltersCorrectly() throws IOException { + String matchQuery = "source=" + INDEX + " | where ts > '2024-03-16 00:00:00' | fields ts"; + assertRoutedToAnalyticsEngine(matchQuery); + JSONObject match = executeQuery(matchQuery); + verifySchema(match, schema("ts", "timestamp")); + verifyDataRows(match, rows("2024-03-16 23:59:59")); + + JSONObject miss = + executeQuery("source=" + INDEX + " | where ts < '2024-03-15 00:00:00' | fields ts"); + Assert.assertEquals( + "Strict comparison should exclude both rows when bound is before any seeded timestamp", + 0, + miss.getJSONArray("datarows").length()); + } + + /** + * {@code year/month/day_of_month/hour} extract calendar fields from the parsed TIMESTAMP, not a + * stringified form. + */ + @Test + public void testTimestampScalarExtractFunctions() throws IOException { + String query = + "source=" + + INDEX + + " | where ts = '2024-03-15 10:30:00'" + + " | eval y = year(ts), m = month(ts), dm = day_of_month(ts), h = hour(ts) " + + "| fields y, m, dm, h"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifySchema( + result, schema("y", "int"), schema("m", "int"), schema("dm", "int"), schema("h", "int")); + verifyDataRows(result, rows(2024, 3, 15, 10)); + } + + /** + * ORDER BY on TIMESTAMP returns rows ascending; schema stays {@code timestamp}, values use space + * separator. + */ + @Test + public void testTimestampOrderByTemporalSemantics() throws IOException { + String query = "source=" + INDEX + " | sort ts | fields ts"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifySchema(result, schema("ts", "timestamp")); + verifyDataRows(result, rows("2024-03-15 10:30:00"), rows("2024-03-16 23:59:59")); + } + + /** + * {@code date_nanos} preserves 9-digit sub-second precision end-to-end (catches micro-truncation + * regressions). + */ + @Test + public void testTimestampNanoPrecisionTrailingNines() throws IOException { + String query = "source=" + INDEX + " | sort ts_nanos | fields ts_nanos"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifySchema(result, schema("ts_nanos", "timestamp")); + verifyDataRows( + result, rows("2024-03-15 10:30:00.123456789"), rows("2024-03-16 23:59:59.999999999")); + } + + /** {@code max(ts)} returns a typed timestamp cell with the documented wire format. */ + @Test + public void testStatsMaxTimestampSpaceFormat() throws IOException { + String query = "source=" + INDEX + " | stats max(ts) as max_ts"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifySchema(result, schema("max_ts", "timestamp")); + verifyDataRows(result, rows("2024-03-16 23:59:59")); + } + + /** + * {@code dc(ts)} on two distinct timestamps returns 2 (AE dedups by temporal identity, not string + * equality). + */ + @Test + public void testStatsCountDistinctTimestamp() throws IOException { + String query = "source=" + INDEX + " | stats dc(ts) as n"; + assertRoutedToAnalyticsEngine(query); + JSONObject result = executeQuery(query); + verifyDataRows(result, rows(2)); + } +}