From 39a865c02461ea971727fed6ccdd081c7ed78795 Mon Sep 17 00:00:00 2001 From: Marcus Henriksson Date: Sun, 7 Dec 2025 14:48:47 +0100 Subject: [PATCH 1/2] fix: Fix issue with left join on nested loops when having pushed down sub query expressions where inner schemas was eliminated and causing ordinal mismatches between compile time and runtime schemas fix: Turned off caching of inner plan in joins. This is not working since the cache is never flushed during a statement execution and hence will yield wrong result if applied in a correlated context --- .../payloadbuilder/core/AQueryResultImpl.java | 2 +- .../core/common/SchemaUtils.java | 35 +++-- .../optimization/ColumnResolver.java | 19 --- .../core/physicalplan/ConstantScan.java | 2 +- .../core/physicalplan/InsertInto.java | 2 +- .../core/physicalplan/NestedLoop.java | 116 +++++++-------- .../physicalplan/OperatorFunctionScan.java | 2 +- .../core/physicalplan/Projection.java | 2 +- .../core/physicalplan/TableScan.java | 19 ++- .../core/planning/QueryPlanner.java | 33 +++-- .../optimization/ColumnResolverTest.java | 8 +- .../core/planning/QueryPlannerTest.java | 133 +++++++----------- .../harnessCases/BaseConstructs.json | 44 +++++- 13 files changed, 212 insertions(+), 205 deletions(-) diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java index 5a4620191..6008409ef 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java @@ -222,7 +222,7 @@ protected void processCurrentPlan(OutputWriter writer, ResultConsumer resultCons Schema schema = currentPlan.getSchema(); boolean currentPlanIsAsterisk = SchemaUtils.isAsterisk(schema); boolean currentPlanHasAsteriskInput = currentPlanIsAsterisk - || SchemaUtils.isAsterisk(schema, true); + || SchemaUtils.originatesFromAsteriskInput(schema); if (resultConsumer != null) { resultConsumer.schema(currentPlanIsAsterisk ? Schema.EMPTY diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/common/SchemaUtils.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/common/SchemaUtils.java index 592d2e271..ba41b169d 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/common/SchemaUtils.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/common/SchemaUtils.java @@ -94,14 +94,22 @@ else if (schema2 == null return new Schema(columns); } - /** Returns true if this schema contains asterisk columns */ + /** + * Returns true if provided schema has columns that originates from any asterisk input. Ie. any column on any level that is of type {@link CoreColumn.Type#ASTERISK} or + * {@link CoreColumn.Type#NAMED_ASTERISK} + */ + public static boolean originatesFromAsteriskInput(Schema schema) + { + return isAsterisk(schema, true, true); + } + + /** Returns true if this schema contains asterisk columns. One or more columns of type {@link CoreColumn.Type#ASTERISK}. */ public static boolean isAsterisk(Schema schema) { - return isAsterisk(schema, false); + return isAsterisk(schema, false, false); } - /** Returns true if this schema contains asterisk columns */ - public static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks) + private static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks, boolean traverseIntoSubSchemas) { int size = schema.getSize(); if (size == 0) @@ -113,7 +121,7 @@ public static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks) { Column column = schema.getColumns() .get(i); - if (isAsterisk(column, includeNamedAsterisks)) + if (isAsterisk(column, includeNamedAsterisks, traverseIntoSubSchemas)) { return true; } @@ -126,13 +134,10 @@ public static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks) */ public static boolean isAsterisk(Column column) { - return isAsterisk(column, false); + return isAsterisk(column, false, false); } - /** - * Returns true if provided column is asterisk. - */ - public static boolean isAsterisk(Column column, boolean includeNamedAsterisks) + private static boolean isAsterisk(Column column, boolean includeNamedAsterisks, boolean traverseIntoSubSchemas) { if (column instanceof CoreColumn cc) { @@ -144,11 +149,15 @@ public static boolean isAsterisk(Column column, boolean includeNamedAsterisks) } // Dig down into complex type schema - if (cc.getType() - .getSchema() != null) + // We also do this for populated columns as those count as asterisk + // if their column is asterisk + if ((traverseIntoSubSchemas + || cc.getColumnType() == CoreColumn.Type.POPULATED) + && cc.getType() + .getSchema() != null) { return isAsterisk(cc.getType() - .getSchema(), includeNamedAsterisks); + .getSchema(), includeNamedAsterisks, traverseIntoSubSchemas); } } return false; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolver.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolver.java index 82cf2d5b1..6e080b341 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolver.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolver.java @@ -699,19 +699,9 @@ public ILogicalPlan visit(OperatorFunctionScan plan, Ctx context) ResolvedType type = pair.getValue() .getType(input.getSchema()); - // If the function type has a sub schema which is asterisk then we treat this column as asterisk - // to easier detect if a schema is asterisk without the need to dig into the sub schema - CoreColumn.Type columnType = CoreColumn.Type.REGULAR; - if (type.getSchema() != null - && SchemaUtils.isAsterisk(type.getSchema(), true)) - { - columnType = CoreColumn.Type.ASTERISK; - } - // Recreate the operator schema // Use type from the resolved input and name from the plans schema Schema schema = Schema.of(CoreColumn.Builder.from(name, type) - .withColumnType(columnType) .withInternal(isInternal) .build()); @@ -1372,15 +1362,6 @@ private ResolveResult resolve(ColumnResolver.Ctx context, QualifiedName original // We only look for direct asterisk columns here boolean isAsterisk = SchemaUtils.getColumnType(schemaColumn) == CoreColumn.Type.ASTERISK; - // We treat internal columns as non asterisk when resolving even if their schema is asterisk due - // to it's input. - // This happens when a subquery expression is pushed down to a join and is getting a generated name (__expr0 etc.) - // and we hit that column, if that one is asterisk we must treat it like a non asterisk match - if (SchemaUtils.isInternal(schemaColumn)) - { - isAsterisk = false; - } - String schemaAlias = p.getKey(); // If the schema doesn't have an alias check to see if the column has a TableSourceReference // attached and if so use the as schemaAlias diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java index ef4b7d681..94d70625b 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java @@ -38,7 +38,7 @@ public ConstantScan(int nodeId, Schema schema, List> rowsExpre this.schema = schema; this.rowsExpressions = requireNonNull(rowsExpressions); this.vector = null; - this.hasAsteriskSchemaOrInput = SchemaUtils.isAsterisk(schema, true); + this.hasAsteriskSchemaOrInput = SchemaUtils.originatesFromAsteriskInput(schema); } public ConstantScan(int nodeId, TupleVector vector) diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java index 603da4eb1..bfe411c2d 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java @@ -30,7 +30,7 @@ public InsertInto(int nodeId, IPhysicalPlan input, List insertColumns, I this.input = requireNonNull(input, "input"); this.insertColumns = insertColumns; this.datasink = requireNonNull(datasink, " datasink"); - this.hasAsteriskSchemaOrInput = SchemaUtils.isAsterisk(input.getSchema(), true); + this.hasAsteriskSchemaOrInput = SchemaUtils.originatesFromAsteriskInput(input.getSchema()); } @Override diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java index c54e85352..4d196a852 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java @@ -93,12 +93,14 @@ public class NestedLoop implements IPhysicalPlan private final boolean switchedInputs; /** The outer schema if this join is a correlated type */ private final Schema outerSchema; - private final boolean isAsteriskOuterSchema; private final Schema schema; private final Schema cartesianSchema; private final boolean isAsteriskSchema; + private final boolean outerSchemaOriginatesFromAsteriskInput; + private final boolean schemaOriginatesFromAsteriskInput; + //@formatter:off private NestedLoop( int nodeId, @@ -124,7 +126,7 @@ private NestedLoop( this.emitEmptyOuterRows = emitEmptyOuterRows; this.switchedInputs = switchedInputs; this.outerSchema = requireNonNull(outerSchema, "outerSchema"); - this.isAsteriskOuterSchema = SchemaUtils.isAsterisk(outerSchema); + this.outerSchemaOriginatesFromAsteriskInput = SchemaUtils.originatesFromAsteriskInput(outerSchema); if (switchedInputs && (condition != null @@ -143,6 +145,7 @@ else if (pushOuterReference this.schema = getSchema(); this.cartesianSchema = getSchema(true); this.isAsteriskSchema = SchemaUtils.isAsterisk(schema); + this.schemaOriginatesFromAsteriskInput = SchemaUtils.originatesFromAsteriskInput(schema); } /** Create an inner join. Logical operations: INNER JOIN */ @@ -318,7 +321,7 @@ else if (populateAlias != null) private TupleVector next; /** Bit set to keep track of outer matches. Used in left joins to know what outer indices to return */ private BitSet outerMatches; - private CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, isAsteriskSchema); + private CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, schemaOriginatesFromAsteriskInput); // The inner schema used when emitting empty outer row, will be the plan schema from start // but if there are inner matches before the un matched ones we switch @@ -593,7 +596,7 @@ private class PopulatingTupleIterator implements TupleIterator private TupleVector next; /** Bit set to keep track of outer matches. Used in left joins to know what outer indices to return */ private BitSet outerMatches; - private CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, isAsteriskSchema); + private CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, schemaOriginatesFromAsteriskInput); // The inner schema used when emitting empty outer row, will be the plan schema from start // but if there are inner matches before the un matched ones we switch @@ -800,7 +803,7 @@ private class LoopTupleIterator implements TupleIterator */ private final OuterTupleVector outerTupleVector; private final RowTupleVector rowTupleVector = new RowTupleVector(); - private final CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, isAsteriskSchema); + private final CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, schemaOriginatesFromAsteriskInput); private TupleVector currentOuter; private TupleVector next; @@ -816,7 +819,7 @@ private class LoopTupleIterator implements TupleIterator this.context = context; this.iterator = iterator; this.nodeData = nodeData; - this.outerTupleVector = new OuterTupleVector(contextOuterTupleVector, outerSchema, isAsteriskOuterSchema); + this.outerTupleVector = new OuterTupleVector(contextOuterTupleVector, outerSchema, outerSchemaOriginatesFromAsteriskInput); } @Override @@ -1091,7 +1094,7 @@ private TupleVector createUnmatchedOuterTuple(ExecutionContext context, Schema i s = SchemaUtils.joinSchema(outer.getSchema(), innerSchema, populateAlias); } - Schema schema = s; + final Schema schema = s; final int outerSize = outer.getSchema() .getSize(); return new TupleVector() @@ -1207,33 +1210,7 @@ void init(TupleVector outer) // We only assert that they are equal if (!outerSchemaIsAsterisk) { - assert (planOuterSchema.equals(this.schema)) : "Planned outer schema should match actual outer schema. Planned: " + planOuterSchema.getColumns() - .stream() - .map(o -> - { - String str = o.toString(); - TableSourceReference tableSource = SchemaUtils.getTableSource(o); - if (tableSource != null) - { - str += " (" + tableSource + ")"; - } - return str; - }) - .collect(joining(", ")) - + ", actual: " - + this.schema.getColumns() - .stream() - .map(o -> - { - String str = o.toString(); - TableSourceReference tableSource = SchemaUtils.getTableSource(o); - if (tableSource != null) - { - str += " (" + tableSource + ")"; - } - return str; - }) - .collect(joining(", ")); + CartesianTupleVector.assertSchemas(planOuterSchema, schema, "outer"); } } @@ -1267,7 +1244,7 @@ public ValueVector getColumn(int column) private static class CartesianTupleVector implements TupleVector { private final Schema cartesianSchema; - private final boolean isAsteriskSchema; + private final boolean schemaOriginatesFromAsteriskInput; private TupleVector outer; private TupleVector inner; @@ -1279,13 +1256,12 @@ private static class CartesianTupleVector implements TupleVector private CartesianColumn[] columns; private int rowCount; private int outerSize; - // private int innerSize; private int innerRowCount; - CartesianTupleVector(Schema cartesianSchema, boolean isAsteriskSchema) + CartesianTupleVector(Schema cartesianSchema, boolean schemaOriginatesFromAsteriskInput) { this.cartesianSchema = cartesianSchema; - this.isAsteriskSchema = isAsteriskSchema; + this.schemaOriginatesFromAsteriskInput = schemaOriginatesFromAsteriskInput; } /** Inits this vector with new outer/inner vectors. Calculates new schemas etc. */ @@ -1306,35 +1282,9 @@ void init(TupleVector outer, TupleVector inner) } // We only assert that they are equal - if (!isAsteriskSchema) + if (!schemaOriginatesFromAsteriskInput) { - assert (cartesianSchema.equals(this.schema)) : "Planned cartesian schema should match actual schema. Planned: " + cartesianSchema.getColumns() - .stream() - .map(o -> - { - String str = o.toString(); - TableSourceReference tableSource = SchemaUtils.getTableSource(o); - if (tableSource != null) - { - str += " (" + tableSource + ")"; - } - return str; - }) - .collect(joining(", ")) - + ", actual: " - + this.schema.getColumns() - .stream() - .map(o -> - { - String str = o.toString(); - TableSourceReference tableSource = SchemaUtils.getTableSource(o); - if (tableSource != null) - { - str += " (" + tableSource + ")"; - } - return str; - }) - .collect(joining(", ")); + assertSchemas(cartesianSchema, schema, "cartesian"); } this.outerSize = outerSchema.getSize(); @@ -1418,6 +1368,40 @@ protected int getRow(int row) return row % innerRowCount; } } + + // Validates that schemas are equal. Used in assert to catch errors in tests + static void assertSchemas(Schema plannedSchema, Schema runtimeSchema, String type) + { + assert (plannedSchema.equals(runtimeSchema)) : "Planned " + type + + " schema should match actual schema. Planned: " + + plannedSchema.getColumns() + .stream() + .map(o -> + { + String str = o.toString(); + TableSourceReference tableSource = SchemaUtils.getTableSource(o); + if (tableSource != null) + { + str += " (" + tableSource + ")"; + } + return str; + }) + .collect(joining(", ")) + + ", actual: " + + runtimeSchema.getColumns() + .stream() + .map(o -> + { + String str = o.toString(); + TableSourceReference tableSource = SchemaUtils.getTableSource(o); + if (tableSource != null) + { + str += " (" + tableSource + ")"; + } + return str; + }) + .collect(joining(", ")); + } } /** Tuple vector that wraps another tuple vector for a single row */ diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java index b18fd3458..85b3adcb2 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java @@ -35,7 +35,7 @@ public OperatorFunctionScan(int nodeId, IPhysicalPlan input, OperatorFunctionInf this.function = requireNonNull(function, "function"); this.catalogAlias = requireNonNull(catalogAlias, "catalogAlias"); this.schema = requireNonNull(schema, "schema"); - this.hasAsteriskSchemaOrInput = SchemaUtils.isAsterisk(schema, true); + this.hasAsteriskSchemaOrInput = SchemaUtils.originatesFromAsteriskInput(schema); if (schema.getColumns() .size() != 1) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java index b6ea18084..48e9d06ea 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java @@ -45,7 +45,7 @@ public Projection(int nodeId, IPhysicalPlan input, Schema schema, List e instanceof AsteriskExpression); this.hasAsteriskSchemaOrInput = hasAsteriskProjection - || SchemaUtils.isAsterisk(schema, true); + || SchemaUtils.originatesFromAsteriskInput(schema); this.parentTableSource = parentTableSource; if (expressions.size() != schema.getSize()) diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java index 5071df61f..a64c82766 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java @@ -212,13 +212,28 @@ static boolean schemaEqualsRegardingTypeAndName(Schema expected, Schema actual) Column vectorColumn = actual.getColumns() .get(i); - if (!schemaColumn.getType() - .equals(vectorColumn.getType()) + if ((schemaColumn.getType() + .getType() != vectorColumn.getType() + .getType()) || !schemaColumn.getName() .equals(vectorColumn.getName())) { return false; } + + Schema schemaSchema = schemaColumn.getType() + .getSchema(); + if (schemaSchema == null) + { + continue; + } + + // Dig down into nested schemas + if (!schemaEqualsRegardingTypeAndName(schemaSchema, vectorColumn.getType() + .getSchema())) + { + return false; + } } } return true; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java index dde11e90e..0b951a2ce 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java @@ -64,7 +64,6 @@ import se.kuseman.payloadbuilder.core.logicalplan.TableSource; import se.kuseman.payloadbuilder.core.logicalplan.optimization.ProjectionMerger; import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeInterceptor; -import se.kuseman.payloadbuilder.core.physicalplan.CachePlan; import se.kuseman.payloadbuilder.core.physicalplan.ExpressionPredicate; import se.kuseman.payloadbuilder.core.physicalplan.HashAggregate; import se.kuseman.payloadbuilder.core.physicalplan.HashMatch; @@ -415,8 +414,10 @@ public IPhysicalPlan visit(Join plan, Context context) throw new IllegalArgumentException("RIGHT joins are not supported"); } + // CSOFF IPhysicalPlan outer = plan.getOuter() .accept(this, context); + // CSON IExpression condition = plan.getCondition(); @@ -501,19 +502,23 @@ public IPhysicalPlan visit(Join plan, Context context) } else { - ValueVector forceNoInnerCacheProperty = context.context.getSession() - .getSystemProperty(QuerySession.FORCE_NO_INNER_CACHE); - boolean forceNoInnerCache = !forceNoInnerCacheProperty.isNull(0) - && forceNoInnerCacheProperty.getBoolean(0); - - // We can cache the inner plan if we have a non correlated plain nested loop - if (!pushOuterReference - && !forceNoInnerCache - && !correlated - && !plan.isSwitchedInputs()) - { - inner = wrapWithAnalyze(context, new CachePlan(context.getNextNodeId(), inner)); - } + // Turned off since cache is broken by design atm. + // It's never reset during a statement and this is not correct + // in many cases when there are nested sub query expressions that is cached when they shouldn't + // + // ValueVector forceNoInnerCacheProperty = context.context.getSession() + // .getSystemProperty(QuerySession.FORCE_NO_INNER_CACHE); + // boolean forceNoInnerCache = !forceNoInnerCacheProperty.isNull(0) + // && forceNoInnerCacheProperty.getBoolean(0); + // + // // We can cache the inner plan if we have a non correlated plain nested loop + // if (!pushOuterReference + // && !forceNoInnerCache + // && !correlated + // && !plan.isSwitchedInputs()) + // { + // inner = wrapWithAnalyze(context, new CachePlan(context.getNextNodeId(), inner)); + // } join = createNestedLoop(plan, context.getNextNodeId(), outer, inner, predicate, pushOuterReference); } diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolverTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolverTest.java index 40b17dfcb..3d71b1919 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolverTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolverTest.java @@ -548,7 +548,7 @@ void test_expand_outer_reference_asterisk_in_sub_query_expression() false, Schema.EMPTY), new OperatorFunctionScan( - Schema.of(ast("__expr0", ResolvedType.table(Schema.of(ast("", "b.*", tableB))), "", null, true)), + Schema.of(col("__expr0", ResolvedType.table(Schema.of(ast("", "b.*", tableB))), true)), projection( ConstantScan.ONE_ROW_EMPTY_SCHEMA, List.of(new AsteriskExpression(QualifiedName.of("b"), null, Set.of(tableB)))), @@ -1020,7 +1020,7 @@ void test_precedence_of_current_outer_matches_asterisk_and_not_asterisk() false, Schema.EMPTY), new OperatorFunctionScan( - Schema.of(ast("__expr0", ResolvedType.table(objectArraySchema), true)), + Schema.of(col("__expr0", ResolvedType.table(objectArraySchema), true)), projection( new ExpressionScan( e_b.withParent(tableB), @@ -1759,7 +1759,7 @@ void test_populate_join_expression_scan_schema_less() false, Schema.EMPTY), new OperatorFunctionScan( - Schema.of(ast("__expr0", ResolvedType.table(subQuerySchema), true)), + Schema.of(col("__expr0", ResolvedType.table(subQuerySchema), true)), projection( new ExpressionScan( e_b.withParent(tableB), @@ -2026,7 +2026,7 @@ void test_that_sub_query_expression_with_table_source_with_unqualified_column_ge new Join( tableScan(schemaSTableA, sTableA), new OperatorFunctionScan( - Schema.of(ast("__expr0", ResolvedType.object(Schema.of(nast("col1", ResolvedType.ANY, tableB), col("col3", Type.Float, sTableA))), true)), + Schema.of(col("__expr0", ResolvedType.object(Schema.of(nast("col1", ResolvedType.ANY, tableB), col("col3", Type.Float, sTableA))), true)), projection( tableScan(schemaB, tableB), List.of( diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java index c195097cf..6419ab807 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java @@ -72,7 +72,6 @@ import se.kuseman.payloadbuilder.core.physicalplan.APhysicalPlanTest; import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeInterceptor; import se.kuseman.payloadbuilder.core.physicalplan.Assert; -import se.kuseman.payloadbuilder.core.physicalplan.CachePlan; import se.kuseman.payloadbuilder.core.physicalplan.ConstantScan; import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan; import se.kuseman.payloadbuilder.core.physicalplan.ExpressionPredicate; @@ -171,7 +170,7 @@ void test_cache_inner_plan_regression() ocre("campaign", variant, CoreColumn.Type.NAMED_ASTERISK)), SystemCatalog.get().getOperatorFunction("object_array"), "sys", - Schema.of(ast("__expr1", ResolvedType.table(campsSchema), true))), + Schema.of(col("__expr1", ResolvedType.table(campsSchema), true))), Set.of(nast("campaign", ResolvedType.ANY, variant)), null, Schema.of(ast("v", ResolvedType.ANY, variant))), @@ -180,7 +179,7 @@ void test_cache_inner_plan_regression() null), SystemCatalog.get().getOperatorFunction("object"), "sys", - Schema.of(ast("__expr0", ResolvedType.object(priceSchema), true))), + Schema.of(col("__expr0", ResolvedType.object(priceSchema), true))), Set.of(nast("campaign", ResolvedType.ANY, variant)), null, Schema.of(ast("v", variant))), @@ -240,9 +239,9 @@ void test_cache_inner_plan_with_nested_complex_structure() col("camps", ResolvedType.table(constantSchema)) ); IPhysicalPlan expected = new Projection( - 10, + 8, NestedLoop.leftJoin( - 9, + 7, new TableScan( 0, variantSchema, @@ -250,33 +249,27 @@ void test_cache_inner_plan_with_nested_complex_structure() "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 8, - new OperatorFunctionScan( - 7, - new Projection( - 6, - NestedLoop.leftJoin( - 5, - new ConstantScan(1, TupleVector.CONSTANT), - new CachePlan( - 4, - new OperatorFunctionScan( - 3, - new ConstantScan(2, TupleVector.of(constantSchema)), - SystemCatalog.get().getOperatorFunction("object_array"), - "sys", - Schema.of(col("__expr1", ResolvedType.table(constantSchema), true))) - ), - null, - false), - Schema.of(col("camps", ResolvedType.table(constantSchema))), - List.of(new AliasExpression(ce("__expr1", 0, ResolvedType.table(constantSchema)), "camps")), - null), - SystemCatalog.get().getOperatorFunction("object"), - "sys", - Schema.of(col("__expr0", ResolvedType.object(priceSchema), true))) - ), + new OperatorFunctionScan( + 6, + new Projection( + 5, + NestedLoop.leftJoin( + 4, + new ConstantScan(1, TupleVector.CONSTANT), + new OperatorFunctionScan( + 3, + new ConstantScan(2, TupleVector.of(constantSchema)), + SystemCatalog.get().getOperatorFunction("object_array"), + "sys", + Schema.of(col("__expr1", ResolvedType.table(constantSchema), true))), + null, + false), + Schema.of(col("camps", ResolvedType.table(constantSchema))), + List.of(new AliasExpression(ce("__expr1", 0, ResolvedType.table(constantSchema)), "camps")), + null), + SystemCatalog.get().getOperatorFunction("object"), + "sys", + Schema.of(col("__expr0", ResolvedType.object(priceSchema), true))), null, false), Schema.of(col("price", ResolvedType.object(priceSchema))), @@ -866,12 +859,9 @@ void test_using_same_table_ref_different_places_non_equi() //@formatter:off IPhysicalPlan expected = NestedLoop.innerJoin( - 3, + 2, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 2, - new TableScan(1, expectedSchemaA1, tableA1, "test", t.scanDataSources.get(1), emptyList()) - ), + new TableScan(1, expectedSchemaA1, tableA1, "test", t.scanDataSources.get(1), emptyList()), new ExpressionPredicate(gt(cre("value", tableA1, ResolvedType.of(Type.Any), CoreColumn.Type.NAMED_ASTERISK), cre("value", tableA, ResolvedType.of(Type.Any), CoreColumn.Type.NAMED_ASTERISK))), null, @@ -917,12 +907,9 @@ void test_outer_apply_non_correlated() Schema expectedSchemaB = Schema.of(ast("b", tableB)); //@formatter:off IPhysicalPlan expected = NestedLoop.leftJoin( - 3, + 2, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()) - ), + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), null, false); //@formatter:on @@ -1079,15 +1066,12 @@ public TableSchema getTableSchema(IExecutionContext context, String catalogAlias //@formatter:off IPhysicalPlan expected = NestedLoop.innerJoin( - 4, + 3, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 3, - new Filter( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), - new ExpressionPredicate(eq(cre("active", tableB, CoreColumn.Type.NAMED_ASTERISK), LiteralBooleanExpression.TRUE))) - ), + new Filter( + 2, + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), + new ExpressionPredicate(eq(cre("active", tableB, CoreColumn.Type.NAMED_ASTERISK), LiteralBooleanExpression.TRUE))), null, false); //@formatter:on @@ -1383,14 +1367,11 @@ public IDatasource getScanDataSource(IQuerySession session, String catalogAlias, //@formatter:off IPhysicalPlan expected = new Sort( - 4, + 3, NestedLoop.innerJoin( - 3, + 2, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()) - ), + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), null, false), asList(sortItem(cre("col", tableA, CoreColumn.Type.NAMED_ASTERISK), Order.DESC, NullOrder.UNDEFINED), @@ -1451,14 +1432,11 @@ public IDatasource getScanDataSource(IQuerySession session, String catalogAlias, //@formatter:off IPhysicalPlan expected = new Sort( - 4, + 3, NestedLoop.innerJoin( - 3, + 2, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()) - ), + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), null, false), asList(sortItem(cre("col2", tableB, CoreColumn.Type.NAMED_ASTERISK), Order.ASC, NullOrder.UNDEFINED))); @@ -1592,24 +1570,21 @@ void test_sub_query_expression() //@formatter:off IPhysicalPlan expected = new Projection( - 6, + 5, NestedLoop.leftJoin( - 5, + 4, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 4, - Assert.maxRowCount( - 3, - new Projection( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), - Schema.of( - nast("__expr0", ResolvedType.ANY, "", tableB, true, "col1") - ), - asList(new AliasExpression(cre("col1", tableB, CoreColumn.Type.NAMED_ASTERISK), "__expr0", true)), - null), - 1) - ), + Assert.maxRowCount( + 3, + new Projection( + 2, + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), + Schema.of( + nast("__expr0", ResolvedType.ANY, "", tableB, true, "col1") + ), + asList(new AliasExpression(cre("col1", tableB, CoreColumn.Type.NAMED_ASTERISK), "__expr0", true)), + null), + 1), null, false), Schema.of( @@ -1702,7 +1677,7 @@ void test_sub_query_expression_with_switched_inputs_with_for() null), SystemCatalog.get().getOperatorFunction("object_array"), "sys", - Schema.of(ast("__expr0", ResolvedType.table(objectArraySchema), true))), + Schema.of(col("__expr0", ResolvedType.table(objectArraySchema), true))), asSet(pop("b", ResolvedType.table(expectedSchemaB), tableB)), null, SchemaUtils.joinSchema(expectedSchemaA, expectedSchemaB, "b")), diff --git a/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json b/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json index ca784787f..34b518063 100644 --- a/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json +++ b/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json @@ -112,14 +112,14 @@ "name": "table", "columns": [ "id", "table", "object", "table2" ], "types": [ - {"type": "String"}, + { "type": "String"}, { "type": "String" }, { "type": "Object", "schema": [ { "name": "key", "type": { "type": "Int" } } ] }, { "type": "Table", "schema": [ { "name": "key", "type": { "type": "Int" } }, { "name": "key2", "type": { "type": "Boolean" } } ] } ], "rows": [ - [ "1", "[{ \"key\": 123, \"value\": 456 },{ \"key\": 789, \"value\": 101112 }]", { "key": 666 }, [{ "key": 4567, "key2": true },{ "key": 891011, "key2": false }] ], - [ "2", "[]", null, [{ "key": 666, "key2": false },{ "key": 1337, "key2": true }] ] + [ "1", "[{ \"key\": 123, \"value\": 456 },{ \"key\": 789, \"value\": 101112 }]", { "key": 666 }, [ { "key": 4567, "key2": true } , { "key": 891011, "key2": false } ] ], + [ "2", "[]", null, [ { "key": 666, "key2": false } , { "key": 1337, "key2": true } ] ] ] } ] @@ -2620,6 +2620,44 @@ [ { "key": "_outer", "value": { "_inner": [ { "key": 666, "key2": false }, { "key": 1337, "key2": true } ] } } ] ] ] + }, + { + "name": "Sub query expressions with no rows", + "description": [ + "Regression test of a query with sub query expressions that returned no rows along with ordinal resolved columns ", + "that got mismatch schemas compile time vs runtime" + ], + "query": [ + "--set @@printplan=true ", + "use complexColumns ", + "SELECT ", + "( ", + " SELECT ", + " ( ", + " SELECT * ", + " FROM (totable(v.table2)) x ", + " WHERE x.key = 666 ", + " FOR OBJECT_ARRAY ", + " ) _outer ", + " ,( ", + " SELECT key * 2 newKey ", + " FROM (totable(v.table2)) x ", + " FOR OBJECT_ARRAY ", + " ) _outer2 ", + " , v.id ", + " FOR OBJECT ", + ") x ", + "FROM ( ", + " select x.table2, x.id ", + " from \"table\" x ", + ") v " + ], + "expectedResultSets": [ + [ + [ { "key": "x", "value": { "_outer": null, "_outer2": [ { "newKey": 9134}, { "newKey": 1782022 } ], "id": "1" } } ], + [ { "key": "x", "value": { "_outer": [ { "key": 666, "key2": false } ], "_outer2": [ { "newKey": 1332}, { "newKey": 2674 } ], "id": "2" } } ] + ] + ] } ] } From 478be772c1bbe161ca2f07f2540890cbbeb6f678 Mon Sep 17 00:00:00 2001 From: Marcus Henriksson Date: Sun, 21 Dec 2025 14:16:02 +0100 Subject: [PATCH 2/2] feat: Add support for setting system properties for generating query plans without explicit grammar keywords --- .../payloadbuilder/core/AQueryResultImpl.java | 27 ++ .../core/execution/QuerySession.java | 17 +- .../payloadbuilder/core/parser/Location.java | 27 +- .../core/parser/QueryParser.java | 12 +- .../physicalplan/APhysicalPlanRewriter.java | 141 +++++++ .../physicalplan/APhysicalPlanVisitor.java | 146 +++++++ .../core/physicalplan/AnalyzeInterceptor.java | 19 +- .../core/physicalplan/AnalyzeVisitor.java | 147 +++++++ .../core/physicalplan/Assert.java | 22 ++ .../core/physicalplan/AssignmentPlan.java | 11 + .../core/physicalplan/CachePlan.java | 11 + .../core/physicalplan/Concatenation.java | 17 + .../core/physicalplan/ConstantScan.java | 8 + .../core/physicalplan/DescribePlan.java | 97 +++-- .../core/physicalplan/DescribeUtils.java | 103 ++++- .../core/physicalplan/ExpressionScan.java | 12 + .../core/physicalplan/Filter.java | 6 + .../core/physicalplan/HashAggregate.java | 26 ++ .../core/physicalplan/HashMatch.java | 22 ++ .../core/physicalplan/IPhysicalPlan.java | 3 + .../physicalplan/IPhysicalPlanVisitor.java | 28 ++ .../core/physicalplan/IndexSeek.java | 6 + .../core/physicalplan/InsertInto.java | 21 + .../core/physicalplan/Limit.java | 16 + .../core/physicalplan/NestedLoop.java | 23 ++ .../physicalplan/OperatorFunctionScan.java | 27 ++ .../core/physicalplan/Projection.java | 6 + .../core/physicalplan/Sort.java | 16 + .../core/physicalplan/TableFunctionScan.java | 6 + .../core/physicalplan/TableScan.java | 6 + .../core/planning/QueryPlanner.java | 56 +-- .../core/planning/StatementPlanner.java | 16 - .../core/planning/StatementRewriter.java | 27 +- .../statement/LogicalSelectStatement.java | 10 +- .../core/statement/PhysicalStatement.java | 10 +- .../core/parser/QueryParserTest.java | 6 +- .../core/physicalplan/APhysicalPlanTest.java | 6 + .../core/physicalplan/ProjectionTest.java | 6 + .../planning/QueryPlannerAnalyzeTest.java | 358 ++++++++++++++++++ .../core/planning/QueryPlannerTest.java | 205 +--------- .../core/test/TestHarnessRunner.java | 27 +- .../harnessCases/BaseConstructs.json | 174 ++++++++- .../harnessCases/TemporaryTables.json | 14 +- 43 files changed, 1576 insertions(+), 368 deletions(-) create mode 100644 payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanRewriter.java create mode 100644 payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanVisitor.java create mode 100644 payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeVisitor.java create mode 100644 payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlanVisitor.java create mode 100644 payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerAnalyzeTest.java diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java index 6008409ef..00f4e6ec8 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java @@ -8,6 +8,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.time.StopWatch; import se.kuseman.payloadbuilder.api.OutputWriter; @@ -25,6 +26,11 @@ import se.kuseman.payloadbuilder.core.execution.OutputWriterUtils; import se.kuseman.payloadbuilder.core.execution.QuerySession; import se.kuseman.payloadbuilder.core.execution.StatementContext; +import se.kuseman.payloadbuilder.core.parser.Location; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeData; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnlayzeType; +import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan; import se.kuseman.payloadbuilder.core.physicalplan.IPhysicalPlan; import se.kuseman.payloadbuilder.core.statement.CacheFlushRemoveStatement; import se.kuseman.payloadbuilder.core.statement.DescribeSelectStatement; @@ -166,6 +172,27 @@ public Void visit(StatementList statement, Void context) public Void visit(PhysicalStatement statement, Void ctx) { currentPlan = statement.getPlan(); + + if (!(currentPlan instanceof DescribePlan)) + { + AnalyzeData analyzeData = AnalyzeVisitor.AnalyzeData.fromSession(session); + if (analyzeData.type() != AnlayzeType.NONE) + { + String queryText = ObjectUtils.getIfNull(statement.getLocation(), Location.EMPTY) + .text(); + IPhysicalPlan analyzePlan = AnalyzeVisitor.describe(currentPlan, analyzeData.type(), analyzeData.format(), queryText); + if (analyzeData.extendOutput()) + { + // Run currentPlan as usual and the add the analyze on top on queue + queue.add(0, new PhysicalStatement(analyzePlan, null)); + } + else + { + currentPlan = analyzePlan; + } + } + } + return null; } diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/execution/QuerySession.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/execution/QuerySession.java index ab65d37b3..cb1386f6a 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/execution/QuerySession.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/execution/QuerySession.java @@ -55,17 +55,26 @@ **/ public class QuerySession implements IQuerySession { - /* System properties */ + /* Compile time System properties */ /** Enable to print logical and physical plans to session print writer */ public static final String PRINT_PLAN = "printplan"; /** Enable to print all logical plans for each optimisation rule. */ public static final String DEBUG_PLAN = "debugplan"; - /** Force a nested loop where default would have been a hash match */ public static final String FORCE_NESTED_LOOP = "force_nested_loop"; + // ** Force no inner cache for non correlated nested loops */ + // public static final String FORCE_NO_INNER_CACHE = "force_no_inner_cache"; + + /* Runtime System properties */ + /** Enable describe of all plans without explicit DESCRIBE key word. This dynamically injects a describe plan on top of all plans. */ + public static final String PLAN_DESCRIBE = "plan_describe"; + /** Enable analyze of all plans without explicit ANALYZE key word. This dynamically injects a analyze plan on top of all plans. */ + public static final String PLAN_ANALYZE = "plan_analyze"; + /** Format of plan output. Only applicable {@link #PLAN_ANALYZE}, {@link #PLAN_DESCRIBE} or explicit DESCRIBE/ANALYZE keyword is set. */ + public static final String PLAN_FORMAT = "plan_format"; + /** Flag that indicates that the query result should be returned along with the plan. Only applicable {@link #PLAN_ANALYZE}, {@link #PLAN_DESCRIBE} or explicit DESCRIBE/ANALYZE keyword is set. */ + public static final String PLAN_EXTENDED_OUTPUT = "plan_extended_output"; - /** Force no inner cache for non correlated nested loops */ - public static final String FORCE_NO_INNER_CACHE = "force_no_inner_cache"; /* End system properties */ /* Compile fields */ diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/Location.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/Location.java index 9905bab72..fde80773c 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/Location.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/Location.java @@ -1,18 +1,26 @@ package se.kuseman.payloadbuilder.core.parser; +import static java.util.Objects.requireNonNull; + import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.tree.TerminalNode; /** A location in query text */ -public record Location(int line, int startOffset, int endOffset) +public record Location(int line, int startOffset, int endOffset, String text) { - public static Location EMPTY = new Location(0, 0, 0); + public static Location EMPTY = new Location(0, 0, 0, ""); + + public Location + { + requireNonNull(text); + } /** Create a location from provided rule context */ public static Location from(ParserRuleContext ctx) { - return new Location(ctx.start.getLine(), ctx.start.getStartIndex(), ctx.stop.getStopIndex() + 1); + return new Location(ctx.start.getLine(), ctx.start.getStartIndex(), ctx.stop.getStopIndex() + 1, ""); } /** Create a location from provided rule context */ @@ -23,6 +31,17 @@ public static Location from(TerminalNode node) node.getSymbol() .getStartIndex(), node.getSymbol() - .getStopIndex() + 1); + .getStopIndex() + 1, + ""); } + + /** Create a location from provided rule context extracting the rule text. */ + public static Location withText(ParserRuleContext ctx) + { + String text = ctx.start.getInputStream() + .getText(new Interval(ctx.start.getStartIndex(), ctx.stop.getStopIndex())); + + return new Location(ctx.start.getLine(), ctx.start.getStartIndex(), ctx.stop.getStopIndex() + 1, text); + } + } diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/QueryParser.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/QueryParser.java index e5bb06776..4d037d110 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/QueryParser.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/QueryParser.java @@ -228,7 +228,7 @@ else if (e instanceof LexerNoViableAltException) } } - Location location = new Location(line, startIndex, stopIndex); + Location location = new Location(line, startIndex, stopIndex, ""); throw new ParseException(msg, location); } }; @@ -469,10 +469,10 @@ public Object visitInsertStatement(InsertStatementContext ctx) inputInput = limit.getInput(); } - input = new LogicalSelectStatement(new Limit(inputInput, topCount), false); + input = new LogicalSelectStatement(new Limit(inputInput, topCount), false, Location.withText(ctx)); } TableName tableName = tableName(ctx.tableName(), ctx.intoOptions); - return new LogicalInsertIntoStatement(input, tableName.catalogAlias, tableName.table, columns, tableName.options, Location.from(ctx)); + return new LogicalInsertIntoStatement(input, tableName.catalogAlias, tableName.table, columns, tableName.options, Location.withText(ctx)); } @Override @@ -490,7 +490,7 @@ public Object visitInsertStatementValue(InsertStatementValueContext ctx) .toList(); ConstantScan constantScan = ConstantScan.createFromRows(expressions, Location.from(ctx.tableValueConstructor())); - return new LogicalSelectStatement(constantScan, false); + return new LogicalSelectStatement(constantScan, false, Location.withText(ctx)); } @Override @@ -531,7 +531,7 @@ public Object visitSelectStatement(SelectStatementContext ctx) plan = wrapTop(plan, ctx); plan = wrapOperatorFunction(plan, ctx); - Statement statement = new LogicalSelectStatement(plan, assignmentSelect); + Statement statement = new LogicalSelectStatement(plan, assignmentSelect, Location.withText(ctx)); if (ctx.into != null) { @@ -541,7 +541,7 @@ public Object visitSelectStatement(SelectStatementContext ctx) } TableName tableName = tableName(ctx.into, ctx.intoOptions); - statement = new LogicalSelectIntoStatement((LogicalSelectStatement) statement, tableName.catalogAlias, tableName.table, tableName.options, Location.from(ctx.into)); + statement = new LogicalSelectIntoStatement((LogicalSelectStatement) statement, tableName.catalogAlias, tableName.table, tableName.options, Location.withText(ctx)); } return statement; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanRewriter.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanRewriter.java new file mode 100644 index 000000000..cad53052a --- /dev/null +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanRewriter.java @@ -0,0 +1,141 @@ +package se.kuseman.payloadbuilder.core.physicalplan; + +import java.util.List; + +/** Base class for rewriters of {@link IPhysicalPlan}'s */ +public abstract class APhysicalPlanRewriter extends APhysicalPlanVisitor +{ + protected IPhysicalPlan visit(IPhysicalPlan plan, C context) + { + return plan.accept(this, context); + } + + @Override + public IPhysicalPlan visit(Projection plan, C context) + { + return new Projection(plan.getNodeId(), visit(plan.getInput(), context), plan.getSchema(), plan.getExpressions(), plan.getParentTableSource()); + } + + @Override + public IPhysicalPlan visit(Sort plan, C context) + { + return new Sort(plan.getNodeId(), visit(plan.getInput(), context), plan.getSortItems()); + } + + @Override + public IPhysicalPlan visit(Filter plan, C context) + { + return new Filter(plan.getNodeId(), visit(plan.getInput(), context), plan.getPredicate()); + } + + @Override + public IPhysicalPlan visit(HashAggregate plan, C context) + { + return new HashAggregate(plan.getNodeId(), visit(plan.getInput(), context), plan.getAggregateExpressions(), plan.getProjectionExpressions(), plan.getParentTableSource()); + } + + @Override + public IPhysicalPlan visit(TableScan plan, C context) + { + // Nothing to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(IndexSeek plan, C context) + { + // Nothing to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(TableFunctionScan plan, C context) + { + // Nothing to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(ExpressionScan plan, C context) + { + // Nothing to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(NestedLoop plan, C context) + { + return NestedLoop.copy(plan, visit(plan.getOuter(), context), visit(plan.getInner(), context)); + } + + @Override + public IPhysicalPlan visit(HashMatch plan, C context) + { + return new HashMatch(plan, visit(plan.getOuter(), context), visit(plan.getInner(), context)); + } + + @Override + public IPhysicalPlan visit(OperatorFunctionScan plan, C context) + { + return new OperatorFunctionScan(plan.getNodeId(), visit(plan.getInput(), context), plan.getFunction(), plan.getCatalogAlias(), plan.getSchema()); + } + + @Override + public IPhysicalPlan visit(ConstantScan plan, C context) + { + // Noting to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(Limit plan, C context) + { + return new Limit(plan.getNodeId(), visit(plan.getInput(), context), plan.getLimitExpression()); + } + + @Override + public IPhysicalPlan visit(Assert plan, C context) + { + return Assert.copy(plan, visit(plan.getInput(), context)); + } + + @Override + public IPhysicalPlan visit(Concatenation plan, C context) + { + List inputs = plan.getInputs() + .stream() + .map(p -> visit(p, context)) + .toList(); + return new Concatenation(plan.getNodeId(), plan.getSchema(), inputs); + } + + @Override + public IPhysicalPlan visit(AnalyzeInterceptor plan, C context) + { + return new AnalyzeInterceptor(plan.getNodeId(), visit(plan.getInput(), context)); + } + + @Override + public IPhysicalPlan visit(AssignmentPlan plan, C context) + { + return new AssignmentPlan(plan.getNodeId(), visit(plan.getInput(), context)); + } + + @Override + public IPhysicalPlan visit(CachePlan plan, C context) + { + return new CachePlan(plan.getNodeId(), visit(plan.getInput(), context)); + } + + @Override + public IPhysicalPlan visit(DescribePlan plan, C context) + { + return new DescribePlan(plan.getNodeId(), visit(plan.getInput(), context), plan.isAnalyze(), plan.getAnalyzeFormat(), plan.getQueryText()); + } + + @Override + public IPhysicalPlan visit(InsertInto plan, C context) + { + return new InsertInto(plan.getNodeId(), visit(plan.getInput(), context), plan.getInsertColumns(), plan.getDatasink()); + } +} diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanVisitor.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanVisitor.java new file mode 100644 index 000000000..9f8a4c5cc --- /dev/null +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanVisitor.java @@ -0,0 +1,146 @@ +package se.kuseman.payloadbuilder.core.physicalplan; + +/** Base class for logical visitors */ +public abstract class APhysicalPlanVisitor implements IPhysicalPlanVisitor +{ + protected T defaultResult(C context) + { + return null; + } + + protected T aggregate(T result, T nextResult) + { + return nextResult; + } + + protected T visitChildren(C context, IPhysicalPlan plan) + { + T result = defaultResult(context); + for (IPhysicalPlan p : plan.getChildren()) + { + T value = p.accept(this, context); + result = aggregate(result, value); + } + return result; + } + + @Override + public T visit(Projection plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Sort plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Filter plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(HashAggregate plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(TableScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(TableFunctionScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(ExpressionScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(NestedLoop plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(HashMatch plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(OperatorFunctionScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(ConstantScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Limit plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Assert plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Concatenation plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(IndexSeek plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(AnalyzeInterceptor plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(AssignmentPlan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(CachePlan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(DescribePlan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(InsertInto plan, C context) + { + return visitChildren(context, plan); + } +} diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeInterceptor.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeInterceptor.java index 8c9efe1e7..5d54d96c8 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeInterceptor.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeInterceptor.java @@ -34,6 +34,12 @@ public IPhysicalPlan getInput() return input; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public List getChildNodes() { @@ -93,18 +99,9 @@ public TupleIterator execute(IExecutionContext context) final AnalyzeData data = context.getStatementContext() .getOrCreateNodeData(nodeId, AnalyzeData::new); data.increaseExecutionCount(); - - if (data.iterator == null) - { - data.iterator = new AnalyzeIterator(data); - } - data.resumeNodeTime(); - final TupleIterator it = input.execute(context); + data.iterator.it = input.execute(context); data.suspenNodeTime(); - - data.iterator.it = it; - return data.iterator; } @@ -241,6 +238,6 @@ public void close() /** Analyze data */ private static class AnalyzeData extends NodeData { - AnalyzeIterator iterator; + AnalyzeIterator iterator = new AnalyzeIterator(this); } } diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeVisitor.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeVisitor.java new file mode 100644 index 000000000..78eaa8175 --- /dev/null +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeVisitor.java @@ -0,0 +1,147 @@ +package se.kuseman.payloadbuilder.core.physicalplan; + +import java.util.ArrayList; +import java.util.List; + +import se.kuseman.payloadbuilder.api.execution.ValueVector; +import se.kuseman.payloadbuilder.core.execution.QuerySession; + +/** Visitor that creates a analyze plan from input plan. */ +public class AnalyzeVisitor extends APhysicalPlanRewriter +{ + private static final AnalyzeVisitor INSTANCE = new AnalyzeVisitor(); + + static class Context + { + int nodeId; + } + + @Override + protected IPhysicalPlan visit(IPhysicalPlan plan, Context context) + { + // Make sure we don't produce analyze plans of already analyzed plans + if (plan instanceof AnalyzeInterceptor) + { + return plan; + } + + IPhysicalPlan analyzePlan = super.visit(plan, context); + return new AnalyzeInterceptor(context.nodeId++, analyzePlan); + } + + /** Describe provided plan with analyze type. */ + public static IPhysicalPlan describe(IPhysicalPlan plan, AnlayzeType analyzeType, AnalyzeFormat analyzeFormat, String queryText) + { + if (plan instanceof DescribePlan) + { + return plan; + } + + boolean isAnalyze = analyzeType == AnlayzeType.ANALYZE; + int nodeId = -1; + if (isAnalyze) + { + plan = analyze(plan); + nodeId = plan.getActualNodeId() + 1; + } + else + { + nodeId = getMaxNodeId(plan); + } + + return new DescribePlan(nodeId, plan, isAnalyze, analyzeFormat, queryText); + } + + /** Create analyze plan from input plan. */ + private static IPhysicalPlan analyze(IPhysicalPlan plan) + { + int maxNodeId = getMaxNodeId(plan); + Context ctx = new Context(); + ctx.nodeId = maxNodeId; + return INSTANCE.visit(plan, ctx); + } + + private static int getMaxNodeId(IPhysicalPlan plan) + { + int maxNodeId = -1; + List queue = new ArrayList<>(); + queue.add(plan); + while (!queue.isEmpty()) + { + IPhysicalPlan current = queue.remove(0); + maxNodeId = Math.max(maxNodeId, current.getNodeId()); + queue.addAll(current.getChildren()); + } + maxNodeId++; + return maxNodeId; + } + + /** Data for performing an analzye of a query statement. */ + public record AnalyzeData(AnlayzeType type, AnalyzeFormat format, boolean extendOutput) + { + /** Extracts analyze data from a session instance. */ + public static AnalyzeData fromSession(QuerySession session) + { + return extract(null, session); + } + + /** Extracts analyze data from a session instance. */ + private static AnalyzeData extract(AnlayzeType type, QuerySession session) + { + boolean extendedOutput = true; + ValueVector prop = session.getSystemProperty(QuerySession.PLAN_EXTENDED_OUTPUT); + if (!prop.isNull(0)) + { + extendedOutput = prop.getBoolean(0); + } + + if (type == null) + { + type = AnlayzeType.NONE; + prop = session.getSystemProperty(QuerySession.PLAN_ANALYZE); + if (prop.getPredicateBoolean(0)) + { + type = AnlayzeType.ANALYZE; + } + if (type == AnlayzeType.NONE) + { + prop = session.getSystemProperty(QuerySession.PLAN_DESCRIBE); + if (prop.getPredicateBoolean(0)) + { + type = AnlayzeType.DESCRIBE; + } + } + } + + AnalyzeFormat format = AnalyzeFormat.TABLE; + prop = session.getSystemProperty(QuerySession.PLAN_FORMAT); + if (!prop.isNull(0)) + { + format = AnalyzeFormat.valueOf(prop.getString(0) + .toString() + .toUpperCase()); + } + + return new AnalyzeData(type, format, extendedOutput); + } + } + + /** Type of analyze. */ + public enum AnlayzeType + { + NONE, + /** Describe plan without execution. */ + DESCRIBE, + /** Describe plan with execution stats. */ + ANALYZE, + } + + /** Output format of analyze. */ + public enum AnalyzeFormat + { + /** JSON tree of plan. Produces a result set with one row and one column. */ + JSON, + /** Regular table vector. */ + TABLE + } +} diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Assert.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Assert.java index ce6e2bbd1..f2c46b0c8 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Assert.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Assert.java @@ -38,12 +38,28 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public Supplier> getPredicateSupplier() + { + return predicateSupplier; + } + @Override public String getName() { return "Assert"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { @@ -138,6 +154,12 @@ public String toString() return "Assert (" + nodeId + "), type: " + predicateSupplier; } + /** Copy assert with new input. */ + public static Assert copy(Assert source, IPhysicalPlan input) + { + return new Assert(source.nodeId, input, source.predicateSupplier); + } + /** Create an assert node that assert max rows from input */ public static Assert maxRowCount(int nodeId, IPhysicalPlan input, int maxRowCount) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AssignmentPlan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AssignmentPlan.java index 645884728..e260118d1 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AssignmentPlan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AssignmentPlan.java @@ -36,12 +36,23 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + @Override public String getName() { return input.getName(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/CachePlan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/CachePlan.java index 789867bc4..f82706835 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/CachePlan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/CachePlan.java @@ -33,6 +33,11 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + @Override public Schema getSchema() { @@ -45,6 +50,12 @@ public String getName() return "Cache"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public TupleIterator execute(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Concatenation.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Concatenation.java index 587a76e8e..849169cb0 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Concatenation.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Concatenation.java @@ -39,6 +39,23 @@ public int getNodeId() return nodeId; } + public List getInputs() + { + return inputs; + } + + @Override + public String getName() + { + return "Concatenation"; + } + + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java index 94d70625b..ced3ef6d5 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java @@ -12,6 +12,7 @@ import java.util.stream.IntStream; import se.kuseman.payloadbuilder.api.catalog.Column; +import se.kuseman.payloadbuilder.api.catalog.IDatasource; import se.kuseman.payloadbuilder.api.catalog.Schema; import se.kuseman.payloadbuilder.api.execution.IExecutionContext; import se.kuseman.payloadbuilder.api.execution.TupleIterator; @@ -62,6 +63,12 @@ public String getName() return "Constant Scan"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { @@ -70,6 +77,7 @@ public Map getDescribeProperties(IExecutionContext context) { if (!Schema.EMPTY.equals(vector.getSchema())) { + properties.put(IDatasource.OUTPUT, DescribeUtils.getOutputColumns(vector.getSchema())); properties.put("Rows", IntStream.range(0, vector.getRowCount()) .mapToObj(i -> "Row " + i + ": " diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribePlan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribePlan.java index e4e840a14..0045f9aaa 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribePlan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribePlan.java @@ -23,6 +23,7 @@ import se.kuseman.payloadbuilder.core.execution.OutputWriterUtils; import se.kuseman.payloadbuilder.core.execution.QuerySession; import se.kuseman.payloadbuilder.core.execution.StatementContext; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeFormat; /** Plan for describing another physical plan and return describable output */ public class DescribePlan implements IPhysicalPlan @@ -31,12 +32,16 @@ public class DescribePlan implements IPhysicalPlan private final int nodeId; private final IPhysicalPlan input; private final boolean analyze; + private final String queryText; + private final AnalyzeFormat format; - public DescribePlan(int nodeId, IPhysicalPlan input, boolean analyze) + public DescribePlan(int nodeId, IPhysicalPlan input, boolean analyze, AnalyzeFormat format, String queryText) { this.nodeId = nodeId; this.input = requireNonNull(input, "input"); this.analyze = analyze; + this.format = requireNonNull(format, "format"); + this.queryText = queryText; } @Override @@ -45,6 +50,26 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public boolean isAnalyze() + { + return analyze; + } + + public AnalyzeFormat getAnalyzeFormat() + { + return format; + } + + public String getQueryText() + { + return queryText; + } + @Override public Schema getSchema() { @@ -57,47 +82,54 @@ public String getName() return "Write Output"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { Map properties = new LinkedHashMap<>(); - NodeData data = context.getStatementContext() - .getNodeData(nodeId); - AnalyzeInterceptor.populateTimings(context, data, this, properties); - properties.put("Allocations", ((ExecutionContext) context).getBufferAllocator() - .getStatistics() - .asObject()); - - Map> catalogStatistics = new HashMap<>(); - QuerySession session = (QuerySession) context.getSession(); - session.getCatalogRegistry() - .getCatalogs() - .forEach(e -> - { - Map statistics = e.getValue() - .getExecutionStatistics(context); - if (statistics.isEmpty()) - { - return; - } - catalogStatistics.put(e.getValue() - .getName(), statistics); - }); + properties.put("Query Text", queryText); - properties.put("Catalog Statistics", catalogStatistics); + if (analyze) + { + NodeData data = context.getStatementContext() + .getNodeData(nodeId); + AnalyzeInterceptor.populateTimings(context, data, this, properties); + properties.put("Allocations", ((ExecutionContext) context).getBufferAllocator() + .getStatistics() + .asObject()); + + Map> catalogStatistics = new HashMap<>(); + QuerySession session = (QuerySession) context.getSession(); + session.getCatalogRegistry() + .getCatalogs() + .forEach(e -> + { + Map statistics = e.getValue() + .getExecutionStatistics(context); + if (statistics.isEmpty()) + { + return; + } + catalogStatistics.put(e.getValue() + .getName(), statistics); + }); + + properties.put("Catalog Statistics", catalogStatistics); + } return properties; } @Override public TupleIterator execute(IExecutionContext context) { - IPhysicalPlan describePlan = input; - // Execute and traverse query in analyze mode before gathering describe data if (analyze) { - describePlan = this; - // Trigger creation of node data to get this nodes timing in output NodeData data = context.getStatementContext() .getOrCreateNodeData(nodeId); @@ -118,20 +150,20 @@ public TupleIterator execute(IExecutionContext context) } finally { + it.close(); + data.suspenNodeTime(); data.increaseRowCount(rowCount); // Populate total query time on every node data long total = TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS); ((StatementContext) context.getStatementContext()).forEachNodeData((i, n) -> n.setTotalQueryTime(total)); - - it.close(); } } ((ExecutionContext) context).getStatementContext() .setOuterTupleVector(new DescribeTupleVector()); - return TupleIterator.singleton(DescribeUtils.getDescribeVector(context, describePlan)); + return TupleIterator.singleton(DescribeUtils.getDescribeVector(context, format, this)); } @Override @@ -163,9 +195,8 @@ else if (obj == this) { return true; } - else if (obj instanceof DescribePlan) + else if (obj instanceof DescribePlan that) { - DescribePlan that = (DescribePlan) obj; return nodeId == that.nodeId && input.equals(that.input) && analyze == that.analyze; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribeUtils.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribeUtils.java index 75b5e19db..c1164582e 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribeUtils.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribeUtils.java @@ -8,9 +8,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.commons.lang3.mutable.MutableInt; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.introspect.VisibilityChecker; +import com.fasterxml.jackson.databind.json.JsonMapper; + import se.kuseman.payloadbuilder.api.catalog.Column; import se.kuseman.payloadbuilder.api.catalog.Column.Type; import se.kuseman.payloadbuilder.api.catalog.ResolvedType; @@ -20,10 +29,19 @@ import se.kuseman.payloadbuilder.api.execution.ValueVector; import se.kuseman.payloadbuilder.core.catalog.CoreColumn; import se.kuseman.payloadbuilder.core.common.DescribableNode; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeFormat; /** Utility class for building a describe result set */ public class DescribeUtils { + private static final String PLB_PLAN_JSON = "PLBPlanJson"; + + private static final JsonMapper MAPPER = JsonMapper.builder() + .visibility(VisibilityChecker.Std.defaultInstance() + .with(Visibility.NONE) + .withFieldVisibility(Visibility.ANY)) + .build(); + public static final String LOGICAL_OPERATOR = "Logical Operator"; public static final String POPULATING = "Populating"; public static final String BATCH_SIZE = "Batch Size"; @@ -65,10 +83,10 @@ public static String getOutputColumns(Schema schema) } /** Generate a describe tuple vector from provided plan */ - static TupleVector getDescribeVector(IExecutionContext context, DescribableNode node) + static TupleVector getDescribeVector(IExecutionContext context, AnalyzeFormat format, DescribableNode node) { final List describeRows = new ArrayList<>(); - collectDescribableRows(context, describeRows, -1, node, 0, "", false); + collectDescribableRows(context, format != AnalyzeFormat.JSON, describeRows, -1, node, 0, "", false); List describeColumns = new ArrayList<>(); Map countByColumn = new HashMap<>(); @@ -106,14 +124,8 @@ static TupleVector getDescribeVector(IExecutionContext context, DescribableNode for (DescribableRow row : describeRows) { Object[] values = new Object[size]; - if (row.nodeId >= 0) - { - values[0] = row.nodeId; - } - if (row.parentNodeId >= 0) - { - values[1] = row.parentNodeId; - } + values[0] = row.nodeId; + values[1] = row.parentNodeId; values[2] = row.name; for (int i = 3; i < size; i++) @@ -129,7 +141,7 @@ static TupleVector getDescribeVector(IExecutionContext context, DescribableNode .map(c -> new Column(c, ResolvedType.of(Type.Any))) .collect(toList())); - return new TupleVector() + TupleVector vector = new TupleVector() { @Override public Schema getSchema() @@ -148,7 +160,6 @@ public ValueVector getColumn(int column) { return new ValueVector() { - @Override public ResolvedType type() { @@ -164,7 +175,16 @@ public int size() @Override public boolean isNull(int row) { - return rows.get(row)[column] == null; + Object value = rows.get(row)[column]; + + if ((column == 0 + || column == 1) + && ((Integer) value) < 0) + { + value = null; + } + + return value == null; } @Override @@ -176,11 +196,56 @@ public Object getAny(int row) }; } }; + + if (format == AnalyzeFormat.TABLE) + { + return vector; + } + else if (format == AnalyzeFormat.JSON) + { + String json = getJson(describeRows); + return TupleVector.of(Schema.of(Column.of(PLB_PLAN_JSON, Type.String)), ValueVector.literalString(json, 1)); + } + + throw new IllegalArgumentException("Unsupported analyze format: " + format); + } + + private static String getJson(List rows) + { + Map rowByNodeId = rows.stream() + .collect(Collectors.toMap(r -> r.nodeId, Function.identity())); + + for (DescribableRow row : rows) + { + DescribableRow parent = rowByNodeId.get(row.parentNodeId); + if (parent == null) + { + continue; + } + else if (parent.children == null) + { + parent.children = new ArrayList<>(); + } + parent.children.add(row); + } + + try + { + return MAPPER.writerWithDefaultPrettyPrinter() + .writeValueAsString(rows.get(0)); + } + catch (JsonProcessingException e) + { + throw new RuntimeException("Error generating query plan as JSON", e); + } } - private static void collectDescribableRows(IExecutionContext context, List rows, int parentNodeId, DescribableNode parent, int pos, String indent, boolean last) + private static void collectDescribableRows(IExecutionContext context, boolean buildTextTree, List rows, int parentNodeId, DescribableNode parent, int pos, String indent, + boolean last) { - rows.add(new DescribableRow(parent.getNodeId(), parentNodeId, indent + "+- " + parent.getName(), parent.getDescribeProperties(context))); + String name = (buildTextTree ? (indent + "+- ") + : "") + parent.getName(); + rows.add(new DescribableRow(parent.getNodeId(), parentNodeId, name, parent.getDescribeProperties(context))); String nextIndent = indent + (last ? " " : "| "); int size = parent.getChildNodes() @@ -189,7 +254,7 @@ private static void collectDescribableRows(IExecutionContext context, List properties; + @JsonProperty + List children; DescribableRow(int nodeId, int parentNodeId, String name, Map properties) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ExpressionScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ExpressionScan.java index 6b41385e2..b117d9cc4 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ExpressionScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ExpressionScan.java @@ -41,6 +41,18 @@ public int getNodeId() return nodeId; } + @Override + public String getName() + { + return "Expression Scan"; + } + + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Filter.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Filter.java index 31ce18a10..479cc7252 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Filter.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Filter.java @@ -51,6 +51,12 @@ public String getName() return "Filter"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + public BiFunction getPredicate() { return predicate; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashAggregate.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashAggregate.java index 321e63b08..60824735a 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashAggregate.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashAggregate.java @@ -87,6 +87,26 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public List getAggregateExpressions() + { + return aggregateExpressions; + } + + public List getProjectionExpressions() + { + return projectionExpressions; + } + + public TableSourceReference getParentTableSource() + { + return parentTableSource; + } + @Override public String getName() { @@ -94,6 +114,12 @@ public String getName() : "Hash Aggregate"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashMatch.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashMatch.java index b409ed93d..201e77d58 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashMatch.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashMatch.java @@ -106,18 +106,40 @@ public HashMatch( this.isAsteriskInnerSchema = SchemaUtils.isAsterisk(inner.getSchema()); } + /** Copy ctor with new inputs */ + public HashMatch(HashMatch source, IPhysicalPlan outer, IPhysicalPlan inner) + { + this(source.nodeId, outer, inner, source.outerHashFunction, source.innerHashFunction, source.condition, source.populateAlias, source.emitEmptyOuterRows, source.pushOuterReference); + } + @Override public int getNodeId() { return nodeId; } + public IPhysicalPlan getOuter() + { + return outer; + } + + public IPhysicalPlan getInner() + { + return inner; + } + @Override public String getName() { return "Hash Match"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlan.java index 3dc9bc545..97fc92bd1 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlan.java @@ -33,6 +33,9 @@ public interface IPhysicalPlan extends DescribableNode /** Return this plans children */ List getChildren(); + /** Accept visitor */ + T accept(IPhysicalPlanVisitor visitor, C context); + /** Returns true if this plan has any writable output else false */ default boolean hasWritableOutput() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlanVisitor.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlanVisitor.java new file mode 100644 index 000000000..c1ce8f0dc --- /dev/null +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlanVisitor.java @@ -0,0 +1,28 @@ +package se.kuseman.payloadbuilder.core.physicalplan; + +/** Visitor definition for {@link IPhysicalPlan}'s */ +public interface IPhysicalPlanVisitor +{ + //@formatter:off + T visit(Projection plan, C context); + T visit(Sort plan, C context); + T visit(Filter plan, C context); + T visit(HashAggregate plan, C context); + T visit(TableScan plan, C context); + T visit(IndexSeek plan, C context); + T visit(TableFunctionScan plan, C context); + T visit(ExpressionScan plan, C context); + T visit(NestedLoop plan, C context); + T visit(HashMatch plan, C context); + T visit(OperatorFunctionScan plan, C context); + T visit(ConstantScan plan, C context); + T visit(Limit plan, C context); + T visit(Assert plan, C context); + T visit(Concatenation plan, C context); + T visit(AnalyzeInterceptor plan, C context); + T visit(AssignmentPlan plan, C context); + T visit(CachePlan plan, C context); + T visit(DescribePlan plan, C context); + T visit(InsertInto plan, C context); + //@formatter:on +} diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IndexSeek.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IndexSeek.java index ccd65665c..7bd1de183 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IndexSeek.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IndexSeek.java @@ -38,6 +38,12 @@ public String getName() return "Index Seek: " + seekPredicate.getIndex(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java index bfe411c2d..bc8a72caf 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java @@ -39,12 +39,33 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public List getInsertColumns() + { + return insertColumns; + } + + public IDatasink getDatasink() + { + return datasink; + } + @Override public String getName() { return "Insert Into"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Limit.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Limit.java index 565d0f4cb..d667b0b19 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Limit.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Limit.java @@ -39,6 +39,16 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public IExpression getLimitExpression() + { + return limitExpression; + } + @Override public Schema getSchema() { @@ -51,6 +61,12 @@ public String getName() return "Limit"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java index 4d196a852..7e1f8158e 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java @@ -148,6 +148,13 @@ else if (pushOuterReference this.schemaOriginatesFromAsteriskInput = SchemaUtils.originatesFromAsteriskInput(schema); } + /** Copy source with new inputs */ + public static NestedLoop copy(NestedLoop source, IPhysicalPlan outer, IPhysicalPlan inner) + { + return new NestedLoop(source.nodeId, outer, inner, source.condition, source.populateAlias, source.outerReferences, source.emitEmptyOuterRows, source.switchedInputs, source.pushOuterReference, + source.outerSchema); + } + /** Create an inner join. Logical operations: INNER JOIN */ public static NestedLoop innerJoin(int nodeId, IPhysicalPlan outer, IPhysicalPlan inner, BiFunction condition, String populateAlias, boolean pushOuterReference) @@ -226,12 +233,28 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getOuter() + { + return outer; + } + + public IPhysicalPlan getInner() + { + return inner; + } + @Override public String getName() { return "Nested Loop"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java index 85b3adcb2..ed348983e 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java @@ -49,6 +49,33 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public OperatorFunctionInfo getFunction() + { + return function; + } + + public String getCatalogAlias() + { + return catalogAlias; + } + + @Override + public String getName() + { + return "Operator Function Scan"; + } + + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java index 48e9d06ea..b0bc01d6a 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java @@ -81,6 +81,12 @@ public String getName() return "Project"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Sort.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Sort.java index c430eeb3c..2fab1ad9b 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Sort.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Sort.java @@ -49,12 +49,28 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public List getSortItems() + { + return sortItems; + } + @Override public String getName() { return "Sort"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableFunctionScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableFunctionScan.java index c061de0c0..dac08bd20 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableFunctionScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableFunctionScan.java @@ -63,6 +63,12 @@ public String getName() return "Function Scan: " + tableSource.getName(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java index a64c82766..1b4da216b 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java @@ -59,6 +59,12 @@ public String getName() return "Scan: " + tableSource.getName(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java index 0b951a2ce..dd5317dc9 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java @@ -63,7 +63,6 @@ import se.kuseman.payloadbuilder.core.logicalplan.TableScan; import se.kuseman.payloadbuilder.core.logicalplan.TableSource; import se.kuseman.payloadbuilder.core.logicalplan.optimization.ProjectionMerger; -import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeInterceptor; import se.kuseman.payloadbuilder.core.physicalplan.ExpressionPredicate; import se.kuseman.payloadbuilder.core.physicalplan.HashAggregate; import se.kuseman.payloadbuilder.core.physicalplan.HashMatch; @@ -101,15 +100,8 @@ public IPhysicalPlan visit(Projection plan, Context context) input = p.getInput(); parentTableSource = p.getParentTableSource(); } - else if (input instanceof AnalyzeInterceptor ai - && ai.getInput() instanceof se.kuseman.payloadbuilder.core.physicalplan.Projection p) - { - expressions = ProjectionMerger.replace(plan.getExpressions(), p.getExpressions()); - input = p.getInput(); - parentTableSource = p.getParentTableSource(); - } - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.Projection(context.getNextNodeId(), input, schema, expressions, parentTableSource)); + return new se.kuseman.payloadbuilder.core.physicalplan.Projection(context.getNextNodeId(), input, schema, expressions, parentTableSource); } @Override @@ -157,7 +149,7 @@ public IPhysicalPlan visit(Sort plan, Context context) return input; } - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.Sort(context.getNextNodeId(), input, plan.getSortItems())); + return new se.kuseman.payloadbuilder.core.physicalplan.Sort(context.getNextNodeId(), input, plan.getSortItems()); } @Override @@ -196,7 +188,7 @@ public IPhysicalPlan visit(Filter plan, Context context) input = filter.getInput(); } - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.Filter(context.getNextNodeId(), input, new ExpressionPredicate(predicate))); + return new se.kuseman.payloadbuilder.core.physicalplan.Filter(context.getNextNodeId(), input, new ExpressionPredicate(predicate)); } @Override @@ -205,7 +197,7 @@ public IPhysicalPlan visit(Aggregate plan, Context context) IPhysicalPlan input = plan.getInput() .accept(this, context); - return wrapWithAnalyze(context, new HashAggregate(context.getNextNodeId(), input, plan.getAggregateExpressions(), plan.getProjectionExpressions(), plan.getParentTableSource())); + return new HashAggregate(context.getNextNodeId(), input, plan.getAggregateExpressions(), plan.getProjectionExpressions(), plan.getParentTableSource()); } @Override @@ -302,10 +294,9 @@ public IPhysicalPlan visit(TableScan plan, Context context) context.topTableScanVisited = true; - return wrapWithAnalyze(context, - seekPredicate != null - ? new se.kuseman.payloadbuilder.core.physicalplan.IndexSeek(nodeId, plan.getSchema(), plan.getTableSource(), catalog.getName(), seekPredicate, dataSource, plan.getOptions()) - : new se.kuseman.payloadbuilder.core.physicalplan.TableScan(nodeId, plan.getSchema(), plan.getTableSource(), catalog.getName(), dataSource, plan.getOptions())); + return seekPredicate != null + ? new se.kuseman.payloadbuilder.core.physicalplan.IndexSeek(nodeId, plan.getSchema(), plan.getTableSource(), catalog.getName(), seekPredicate, dataSource, plan.getOptions()) + : new se.kuseman.payloadbuilder.core.physicalplan.TableScan(nodeId, plan.getSchema(), plan.getTableSource(), catalog.getName(), dataSource, plan.getOptions()); } @Override @@ -324,14 +315,14 @@ public IPhysicalPlan visit(TableFunctionScan plan, Context context) .getCatalog(catalogAlias); TableFunctionInfo functionInfo = pair.getValue(); - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.TableFunctionScan(context.getNextNodeId(), plan.getSchema(), plan.getTableSource(), catalogAlias, - catalog.getName(), functionInfo, plan.getArguments(), plan.getOptions())); + return new se.kuseman.payloadbuilder.core.physicalplan.TableFunctionScan(context.getNextNodeId(), plan.getSchema(), plan.getTableSource(), catalogAlias, catalog.getName(), functionInfo, + plan.getArguments(), plan.getOptions()); } @Override public IPhysicalPlan visit(ExpressionScan plan, Context context) { - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ExpressionScan(context.getNextNodeId(), plan.getTableSource(), plan.getSchema(), plan.getExpression())); + return new se.kuseman.payloadbuilder.core.physicalplan.ExpressionScan(context.getNextNodeId(), plan.getTableSource(), plan.getSchema(), plan.getExpression()); } @Override @@ -523,7 +514,7 @@ public IPhysicalPlan visit(Join plan, Context context) join = createNestedLoop(plan, context.getNextNodeId(), outer, inner, predicate, pushOuterReference); } - return wrapWithAnalyze(context, join); + return join; } @Override @@ -537,7 +528,7 @@ public IPhysicalPlan visit(OperatorFunctionScan plan, Context context) String catalogAlias = pair.getKey(); OperatorFunctionInfo functionInfo = pair.getValue(); - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.OperatorFunctionScan(context.getNextNodeId(), input, functionInfo, catalogAlias, plan.getSchema())); + return new se.kuseman.payloadbuilder.core.physicalplan.OperatorFunctionScan(context.getNextNodeId(), input, functionInfo, catalogAlias, plan.getSchema()); } @Override @@ -545,18 +536,18 @@ public IPhysicalPlan visit(ConstantScan plan, Context context) { if (ConstantScan.ONE_ROW_EMPTY_SCHEMA.equals(plan)) { - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.CONSTANT)); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.CONSTANT); } else if (ConstantScan.ZERO_ROWS_EMPTY_SCHEMA.equals(plan)) { - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.EMPTY)); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.EMPTY); } // Zero row scan with a schema if (plan.getRowsExpressions() .isEmpty()) { - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.of(plan.getSchema()))); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.of(plan.getSchema())); } // Strip all alias expression here since they are not needed anymore @@ -575,10 +566,10 @@ else if (ConstantScan.ZERO_ROWS_EMPTY_SCHEMA.equals(plan)) .allMatch(IExpression::isConstant))) { TupleVector vector = se.kuseman.payloadbuilder.core.physicalplan.ConstantScan.vectorize(plan.getSchema(), rowsExpressions, context.context, false); - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), vector)); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), vector); } - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), plan.getSchema(), rowsExpressions)); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), plan.getSchema(), rowsExpressions); } @Override @@ -586,7 +577,7 @@ public IPhysicalPlan visit(Limit plan, Context context) { IPhysicalPlan input = plan.getInput() .accept(this, context); - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.Limit(context.getNextNodeId(), input, plan.getLimitExpression())); + return new se.kuseman.payloadbuilder.core.physicalplan.Limit(context.getNextNodeId(), input, plan.getLimitExpression()); } @Override @@ -594,7 +585,7 @@ public IPhysicalPlan visit(MaxRowCountAssert plan, Context context) { IPhysicalPlan input = plan.getInput() .accept(this, context); - return wrapWithAnalyze(context, se.kuseman.payloadbuilder.core.physicalplan.Assert.maxRowCount(context.getNextNodeId(), input, plan.getMaxRowCount())); + return se.kuseman.payloadbuilder.core.physicalplan.Assert.maxRowCount(context.getNextNodeId(), input, plan.getMaxRowCount()); } @Override @@ -683,15 +674,6 @@ private TableSource getTopTableSource(ILogicalPlan plan, boolean skipSubQuery) return null; } - static IPhysicalPlan wrapWithAnalyze(Context context, IPhysicalPlan plan) - { - if (context.analyze) - { - return new AnalyzeInterceptor(context.getNextNodeId(), plan); - } - return plan; - } - /** Visitor that collects table source references */ static class TableSourceReferenceCollector extends AExpressionVisitor> { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementPlanner.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementPlanner.java index afef4216d..5fb02a556 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementPlanner.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementPlanner.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Map; -import se.kuseman.payloadbuilder.api.QualifiedName; import se.kuseman.payloadbuilder.api.catalog.ISortItem; import se.kuseman.payloadbuilder.api.catalog.TableSchema; import se.kuseman.payloadbuilder.api.execution.IExecutionContext; @@ -34,13 +33,8 @@ static class Context boolean joinPreserveOuterOrder; boolean topTableScanVisited; - Map schemaByTempTable = new HashMap<>(); - Map schemaByTableSource = new HashMap<>(); - /** Flag indicating that next plan to be generated is an analzye */ - boolean analyze; - /** Field with the last created logical plan. Used when holding schema information for temp tables etc. */ ILogicalPlan currentLogicalPlan; @@ -80,16 +74,6 @@ public static QueryStatement plan(QuerySession session, QueryStatement query) ExecutionContext context = new ExecutionContext(session); Context ctx = new Context(context); - // Add existing session tables into context - // This is used when working in a statefull session system like Queryeer - // where the session is reused across executions. - session.getTemporaryTables() - .forEach(e -> ctx.schemaByTempTable.put(e.getKey(), new TableSchema(e.getValue() - .getTupleVector() - .getSchema(), - e.getValue() - .getIndices()))); - return new QueryStatement(query.getStatements() .stream() .map(s -> s.accept(STATEMENT_REWRITER, ctx)) diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementRewriter.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementRewriter.java index d26df1289..e04c28159 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementRewriter.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementRewriter.java @@ -9,6 +9,8 @@ import java.util.List; +import org.apache.commons.lang3.ObjectUtils; + import se.kuseman.payloadbuilder.api.QualifiedName; import se.kuseman.payloadbuilder.api.catalog.Catalog; import se.kuseman.payloadbuilder.api.catalog.IDatasink; @@ -36,9 +38,12 @@ import se.kuseman.payloadbuilder.core.logicalplan.Sort; import se.kuseman.payloadbuilder.core.logicalplan.TableScan; import se.kuseman.payloadbuilder.core.logicalplan.optimization.LogicalPlanOptimizer; +import se.kuseman.payloadbuilder.core.parser.Location; import se.kuseman.payloadbuilder.core.parser.ParseException; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeFormat; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnlayzeType; import se.kuseman.payloadbuilder.core.physicalplan.AssignmentPlan; -import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan; import se.kuseman.payloadbuilder.core.physicalplan.IPhysicalPlan; import se.kuseman.payloadbuilder.core.physicalplan.InsertInto; import se.kuseman.payloadbuilder.core.planning.StatementPlanner.Context; @@ -115,11 +120,15 @@ public Statement visit(UseStatement statement, Context context) @Override public Statement visit(DescribeSelectStatement statement, Context context) { - context.analyze = statement.isAnalyze(); PhysicalStatement physicalStatement = (PhysicalStatement) statement.getStatement() .accept(this, context); - context.analyze = false; - return new PhysicalStatement(new DescribePlan(context.getNextNodeId(), physicalStatement.getPlan(), statement.isAnalyze())); + + IPhysicalPlan plan = physicalStatement.getPlan(); + AnalyzeVisitor.AnlayzeType type = statement.isAnalyze() ? AnlayzeType.ANALYZE + : AnlayzeType.DESCRIBE; + String queryText = ObjectUtils.getIfNull(physicalStatement.getLocation(), Location.EMPTY) + .text(); + return new PhysicalStatement(AnalyzeVisitor.describe(plan, type, AnalyzeFormat.TABLE, queryText), null); } @Override @@ -164,7 +173,7 @@ public Statement visit(ShowStatement statement, Context context) new LiteralStringExpression(UTF8String.EMPTY), new LiteralStringExpression(UTF8String.EMPTY)), null), - new Sort(systemFunctionsScan, sortItems)), null), false).accept(this, context); + new Sort(systemFunctionsScan, sortItems)), null), false, Location.EMPTY).accept(this, context); //@formatter:on } @@ -183,8 +192,8 @@ public Statement visit(ShowStatement statement, Context context) QualifiedName qname = isBlank(catalog) ? QualifiedName.of(tableName) : QualifiedName.of(catalog, tableName); TableSourceReference tableSourceRef = new TableSourceReference(2, TableSourceReference.Type.TABLE, "sys", qname, "t"); - return new LogicalSelectStatement(new TableScan(TableSchema.EMPTY, tableSourceRef, se.kuseman.payloadbuilder.api.catalog.DatasourceData.Projection.ALL, emptyList(), null), false).accept(this, - context); + return new LogicalSelectStatement(new TableScan(TableSchema.EMPTY, tableSourceRef, se.kuseman.payloadbuilder.api.catalog.DatasourceData.Projection.ALL, emptyList(), Location.EMPTY), false, + null).accept(this, context); } @Override @@ -264,7 +273,7 @@ private Statement createInsertStatement(LogicalSelectIntoStatement statement, Co : catalog.getInsertIntoSink(context.getSession(), catalogAlias, statement.getTable(), new InsertIntoData(nodeId, schema, options, insertColumns)); //@formatter:on - return new PhysicalStatement(QueryPlanner.wrapWithAnalyze(context, new InsertInto(nodeId, input.getPlan(), insertColumns, sink))); + return new PhysicalStatement(new InsertInto(nodeId, input.getPlan(), insertColumns, sink), statement.getLocation()); } private PhysicalStatement createPhysicalPlan(LogicalSelectStatement statement, Context context) @@ -304,6 +313,6 @@ private PhysicalStatement createPhysicalPlan(LogicalSelectStatement statement, C } // Create a physical plan from the logical - return new PhysicalStatement(physicalPlan); + return new PhysicalStatement(physicalPlan, statement.getLocation()); } } diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/LogicalSelectStatement.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/LogicalSelectStatement.java index 755c7e4f9..5bb5e311c 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/LogicalSelectStatement.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/LogicalSelectStatement.java @@ -3,17 +3,20 @@ import static java.util.Objects.requireNonNull; import se.kuseman.payloadbuilder.core.logicalplan.ILogicalPlan; +import se.kuseman.payloadbuilder.core.parser.Location; /** Logical select statement. Statement in the planning phase. Will be transformed in to a {@link PhysicalStatement} further down the line */ public class LogicalSelectStatement extends Statement { private final ILogicalPlan select; private final boolean assignmentSelect; + private final Location location; - public LogicalSelectStatement(ILogicalPlan select, boolean assignmentSelect) + public LogicalSelectStatement(ILogicalPlan select, boolean assignmentSelect, Location location) { this.select = requireNonNull(select, "select"); this.assignmentSelect = assignmentSelect; + this.location = location; } public ILogicalPlan getSelect() @@ -26,6 +29,11 @@ public boolean isAssignmentSelect() return assignmentSelect; } + public Location getLocation() + { + return location; + } + @Override public TR accept(StatementVisitor visitor, TC context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/PhysicalStatement.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/PhysicalStatement.java index 76244f68e..5ac24c9a3 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/PhysicalStatement.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/PhysicalStatement.java @@ -2,16 +2,19 @@ import static java.util.Objects.requireNonNull; +import se.kuseman.payloadbuilder.core.parser.Location; import se.kuseman.payloadbuilder.core.physicalplan.IPhysicalPlan; /** Physical statement that wraps a {@link IPhysicalPlan}. */ public class PhysicalStatement extends Statement { private final IPhysicalPlan plan; + private final Location location; - public PhysicalStatement(IPhysicalPlan plan) + public PhysicalStatement(IPhysicalPlan plan, Location location) { this.plan = requireNonNull(plan, "plan"); + this.location = location; } public IPhysicalPlan getPlan() @@ -19,6 +22,11 @@ public IPhysicalPlan getPlan() return plan; } + public Location getLocation() + { + return location; + } + @Override public TR accept(StatementVisitor visitor, TC context) { diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/parser/QueryParserTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/parser/QueryParserTest.java index 1d81dd76a..597423a4c 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/parser/QueryParserTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/parser/QueryParserTest.java @@ -304,7 +304,7 @@ void test_subquery_expression() null), null)), null), - false); + false, null); //@formatter:on Statement actual = s("select a.col, (select a.* from open_table(a) a for objectarray) from \"table\" a where a.col > 10"); @@ -667,7 +667,7 @@ void test_insert_into() void test_select() { // Selects without table source - assertEquals(new LogicalSelectStatement(ConstantScan.create(asList(litInt(1)), null), false), assertSelect("select 1")); + assertEquals(new LogicalSelectStatement(ConstantScan.create(asList(litInt(1)), null), false, null), assertSelect("select 1")); assertSelect("select 1 where false"); assertSelect("select 1 order by 1"); assertSelect("select top 10 1"); @@ -682,7 +682,7 @@ void test_select() null), List.of(new AsteriskExpression(null)), null), - false), assertSelect("select * from (a.b) a")); + false, null), assertSelect("select * from (a.b) a")); //@formatter:on assertSelectFail(ParseException.class, "Expression scans cannot have options", "select * from (a.b) a with (a=123)"); diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanTest.java index 842a76cb6..c7be39d9d 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanTest.java @@ -149,6 +149,12 @@ public List getChildren() return emptyList(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + throw new RuntimeException("Cannot be visited"); + } + @Override public TupleIterator execute(IExecutionContext context) { diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/ProjectionTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/ProjectionTest.java index c4b36baeb..7f281d82b 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/ProjectionTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/ProjectionTest.java @@ -151,6 +151,12 @@ public int getNodeId() return 0; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + throw new RuntimeException("Cannot be visited"); + } + @Override public List getChildren() { diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerAnalyzeTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerAnalyzeTest.java new file mode 100644 index 000000000..f2a540718 --- /dev/null +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerAnalyzeTest.java @@ -0,0 +1,358 @@ +package se.kuseman.payloadbuilder.core.planning; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static se.kuseman.payloadbuilder.api.utils.MapUtils.entry; +import static se.kuseman.payloadbuilder.api.utils.MapUtils.ofEntries; +import static se.kuseman.payloadbuilder.core.planning.QueryPlannerTest.parse; + +import java.util.List; + +import org.apache.commons.lang3.tuple.Triple; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import se.kuseman.payloadbuilder.api.QualifiedName; +import se.kuseman.payloadbuilder.api.catalog.Column.Type; +import se.kuseman.payloadbuilder.api.catalog.FunctionInfo; +import se.kuseman.payloadbuilder.api.catalog.IDatasource; +import se.kuseman.payloadbuilder.api.catalog.IPredicate; +import se.kuseman.payloadbuilder.api.catalog.ResolvedType; +import se.kuseman.payloadbuilder.api.catalog.ScalarFunctionInfo; +import se.kuseman.payloadbuilder.api.catalog.Schema; +import se.kuseman.payloadbuilder.core.catalog.CoreColumn; +import se.kuseman.payloadbuilder.core.catalog.TableSourceReference; +import se.kuseman.payloadbuilder.core.expression.FunctionCallExpression; +import se.kuseman.payloadbuilder.core.expression.InExpression; +import se.kuseman.payloadbuilder.core.expression.LikeExpression; +import se.kuseman.payloadbuilder.core.expression.LiteralStringExpression; +import se.kuseman.payloadbuilder.core.expression.NullPredicateExpression; +import se.kuseman.payloadbuilder.core.physicalplan.APhysicalPlanTest; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeInterceptor; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeFormat; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnlayzeType; +import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan; +import se.kuseman.payloadbuilder.core.physicalplan.ExpressionPredicate; +import se.kuseman.payloadbuilder.core.physicalplan.Filter; +import se.kuseman.payloadbuilder.core.physicalplan.IPhysicalPlan; +import se.kuseman.payloadbuilder.core.physicalplan.Projection; +import se.kuseman.payloadbuilder.core.physicalplan.TableScan; +import se.kuseman.payloadbuilder.core.planning.QueryPlannerTest.TestCatalog; +import se.kuseman.payloadbuilder.core.statement.PhysicalStatement; +import se.kuseman.payloadbuilder.core.statement.QueryStatement; +import se.kuseman.payloadbuilder.core.utils.CollectionUtils; + +/** Query planner test with analyze option. */ +class QueryPlannerAnalyzeTest extends APhysicalPlanTest +{ + @BeforeEach + void setup() + { + session.setDefaultCatalogAlias("t"); + } + + @Test + void test_analyze_dont_describe_describe_plans() + { + //@formatter:off + IPhysicalPlan plan = new DescribePlan( + 11, + new AnalyzeInterceptor( + 10, + new TableScan( + 0, + Schema.of(ast("", table)), + table, + "test", + mock(IDatasource.class), + emptyList())), + true, + AnalyzeFormat.JSON, + ""); + //@formatter:on + assertSame(plan, AnalyzeVisitor.describe(plan, AnlayzeType.ANALYZE, AnalyzeFormat.JSON, "")); + } + + @Test + void test_analyze_dont_intercept_analyze_interceptors() + { + //@formatter:off + IPhysicalPlan plan = new DescribePlan( + 11, + new AnalyzeInterceptor( + 10, + new TableScan( + 0, + Schema.of(ast("", table)), + table, + "test", + mock(IDatasource.class), + emptyList())), + true, + AnalyzeFormat.JSON, + ""); + //@formatter:on + assertSame(plan.getChildren() + .get(0), + AnalyzeVisitor.describe(plan.getChildren() + .get(0), AnlayzeType.ANALYZE, AnalyzeFormat.JSON, "") + .getChildren() + .get(0)); + } + + @Test + void test_describe_tableScan() + { + //@formatter:off + String query = """ + describe + select * + from tableA + """; + //@formatter:on + + TestCatalog t = new TestCatalog(emptyMap()); + catalogRegistry.registerCatalog("t", t); + + QueryStatement queryStatement = parse(query); + queryStatement = StatementPlanner.plan(session, queryStatement); + + IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() + .get(0)).getPlan(); + + TableSourceReference table = new TableSourceReference(0, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), ""); + + //@formatter:off + IPhysicalPlan expected = + new DescribePlan( + 1, + new TableScan( + 0, + Schema.of(ast("", table)), + table, + "test", + t.scanDataSources.get(0), + emptyList()), + false, AnalyzeFormat.TABLE, ""); + //@formatter:on + + // System.out.println(actual.print(0)); + // System.out.println(expected.print(0)); + + Assertions.assertThat(actual) + .usingRecursiveComparison() + .ignoringFields("queryText") + .isEqualTo(expected); + + assertEquals(expected, actual); + } + + @Test + void test_sub_query_filter_elimination_analyze() + { + // Verify that the nested filters are merged when the subquery are eliminated + //@formatter:off + String query = """ + analyze + select * + from + ( + select * + from tableA a + where a.col1 > 10 + ) x + where x.col > 20 + """; + //@formatter:on + + TestCatalog t = new TestCatalog(emptyMap()); + catalogRegistry.registerCatalog("t", t); + + QueryStatement queryStatement = parse(query); + queryStatement = StatementPlanner.plan(session, queryStatement); + + IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() + .get(0)).getPlan(); + + TableSourceReference tableA = new TableSourceReference(1, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), "a"); + + Schema expectedSchemaA = Schema.of(ast("a", tableA)); + + //@formatter:off + IPhysicalPlan expected = + new DescribePlan(4, + new AnalyzeInterceptor( + 3, + new Filter( + 1, + new AnalyzeInterceptor( + 2, + new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()) + ), + new ExpressionPredicate(and(gt(cre("col1", tableA, CoreColumn.Type.NAMED_ASTERISK), intLit(10)), + gt(cre("col", tableA, CoreColumn.Type.NAMED_ASTERISK), intLit(20)))) + )), + true, AnalyzeFormat.TABLE, ""); + //@formatter:on + + // System.out.println(actual.print(0)); + // System.out.println(expected.print(0)); + + Assertions.assertThat(actual) + .usingRecursiveComparison() + .ignoringFields("queryText") + .isEqualTo(expected); + + assertEquals(expected, actual); + } + + @Test + void test_sub_query_projection_elimination_analyze() + { + // Verify that the nested projections are merged when the subquery are eliminated + //@formatter:off + String query = """ + analyze + select x.col1, x.col2, x.col3 + x.col2 + from + ( + select col1, col2, col3 + from tableA + ) x + """; + //@formatter:on + + TestCatalog t = new TestCatalog(emptyMap()); + catalogRegistry.registerCatalog("t", t); + + QueryStatement queryStatement = parse(query); + queryStatement = StatementPlanner.plan(session, queryStatement); + + IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() + .get(0)).getPlan(); + + TableSourceReference tableA = new TableSourceReference(1, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), ""); + TableSourceReference subQueryX = new TableSourceReference(0, TableSourceReference.Type.SUBQUERY, "", QualifiedName.of("x"), "x"); + Schema expectedSchemaA = Schema.of(ast("", tableA)); + + //@formatter:off + IPhysicalPlan expected = + new DescribePlan( + 5, + new AnalyzeInterceptor( + 4, + new Projection( + 2, + new AnalyzeInterceptor( + 3, + new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()) + ), + Schema.of( + nast("col1", ResolvedType.ANY, tableA), + nast("col2", ResolvedType.ANY, tableA), + col("", Type.Any, "col3 + col2") + ), + List.of(cre("col1", tableA, CoreColumn.Type.NAMED_ASTERISK), cre("col2", tableA, CoreColumn.Type.NAMED_ASTERISK), + add(cre("col3", tableA, CoreColumn.Type.NAMED_ASTERISK), cre("col2", tableA, CoreColumn.Type.NAMED_ASTERISK))), + subQueryX + ) + ), + true, AnalyzeFormat.TABLE, ""); + //@formatter:on + + // System.out.println(actual.print(0)); + // System.out.println(expected.print(0)); + + Assertions.assertThat(actual) + .usingRecursiveComparison() + .ignoringFields("queryText") + .isEqualTo(expected); + + assertEquals(expected, actual); + } + + @Test + void test_predicate_pushdown_with_analyze() + { + //@formatter:off + String query = "" + + "analyze select * " + + "from \"table\" " + + "where col = 10 " // Comparison + + "and col2 in (10,20) " // In + + "and col3 like 'string' " // Like + + "and col4 is null " // Null + + "and someFunc() "; // Function call + + //@formatter:on + + // Capture col and "" (someFunc) + ScalarFunctionInfo someFunc = new ScalarFunctionInfo("someFunc", FunctionInfo.FunctionType.SCALAR) + { + }; + TestCatalog catalog = new TestCatalog(ofEntries(entry(QualifiedName.of("table"), CollectionUtils.asSet("col", "")))) + { + { + registerFunction(someFunc); + } + }; + + catalogRegistry.registerCatalog("t", catalog); + + QueryStatement queryStatement = parse(query); + queryStatement = StatementPlanner.plan(session, queryStatement); + + IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() + .get(0)).getPlan(); + + TableSourceReference table = new TableSourceReference(0, TableSourceReference.Type.TABLE, "", QualifiedName.of("table"), ""); + + //@formatter:off + Schema expectedSchema = Schema.of(ast("", table)); + + IPhysicalPlan expected = new DescribePlan( + 4, + new AnalyzeInterceptor( + 3, + new Filter( + 1, + new AnalyzeInterceptor( + 2, + new TableScan(0, expectedSchema, table, "test", catalog.scanDataSources.get(0), emptyList()) + ), + new ExpressionPredicate( + and( + and( + new InExpression(cre("col2", table, CoreColumn.Type.NAMED_ASTERISK), asList(intLit(10), intLit(20)), false), + new LikeExpression(cre("col3", table, CoreColumn.Type.NAMED_ASTERISK), new LiteralStringExpression("string"), false, null) + ), + new NullPredicateExpression(cre("col4", table, CoreColumn.Type.NAMED_ASTERISK), false))) + )), + true, AnalyzeFormat.TABLE, ""); + //@formatter:on + + //@formatter:off + assertEquals(1, catalog.consumedPredicate.size()); + assertEquals(asList( + Triple.of(null, IPredicate.Type.FUNCTION_CALL, asList(new FunctionCallExpression("t", someFunc, null, asList()))), + Triple.of(QualifiedName.of("col"), IPredicate.Type.COMPARISION, asList(intLit(10))) + ), catalog.consumedPredicate.get(QualifiedName.of("table"))); + //@formatter:on + + // System.out.println(actual.print(0)); + // System.out.println(expected.print(0)); + + Assertions.assertThat(actual) + .usingRecursiveComparison() + .ignoringFields("queryText") + .isEqualTo(expected); + + assertEquals(expected, actual); + } +} diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java index 6419ab807..3f6f54c9f 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java @@ -70,10 +70,8 @@ import se.kuseman.payloadbuilder.core.expression.VariableExpression; import se.kuseman.payloadbuilder.core.parser.Location; import se.kuseman.payloadbuilder.core.physicalplan.APhysicalPlanTest; -import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeInterceptor; import se.kuseman.payloadbuilder.core.physicalplan.Assert; import se.kuseman.payloadbuilder.core.physicalplan.ConstantScan; -import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan; import se.kuseman.payloadbuilder.core.physicalplan.ExpressionPredicate; import se.kuseman.payloadbuilder.core.physicalplan.ExpressionScan; import se.kuseman.payloadbuilder.core.physicalplan.Filter; @@ -335,63 +333,6 @@ void test_sub_query_filter_elimination() assertEquals(expected, actual); } - @Test - void test_sub_query_filter_elimination_analyze() - { - // Verify that the nested filters are merged when the subquery are eliminated - //@formatter:off - String query = """ - analyze - select * - from - ( - select * - from tableA a - where a.col1 > 10 - ) x - where x.col > 20 - """; - //@formatter:on - - TestCatalog t = new TestCatalog(emptyMap()); - catalogRegistry.registerCatalog("t", t); - - QueryStatement queryStatement = parse(query); - queryStatement = StatementPlanner.plan(session, queryStatement); - - IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() - .get(0)).getPlan(); - - TableSourceReference tableA = new TableSourceReference(1, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), "a"); - - Schema expectedSchemaA = Schema.of(ast("a", tableA)); - - //@formatter:off - IPhysicalPlan expected = new DescribePlan(4, - new AnalyzeInterceptor( - 3, - new Filter( - 2, - new AnalyzeInterceptor( - 1, - new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()) - ), - new ExpressionPredicate(and(gt(cre("col1", tableA, CoreColumn.Type.NAMED_ASTERISK), intLit(10)), - gt(cre("col", tableA, CoreColumn.Type.NAMED_ASTERISK), intLit(20)))) - )), - true); - //@formatter:on - - // System.out.println(actual.print(0)); - // System.out.println(expected.print(0)); - - Assertions.assertThat(actual) - .usingRecursiveComparison() - .isEqualTo(expected); - - assertEquals(expected, actual); - } - @Test void test_nested_sub_query_projection_elimination() { @@ -505,70 +446,6 @@ void test_sub_query_projection_elimination() assertEquals(expected, actual); } - @Test - void test_sub_query_projection_elimination_analyze() - { - // Verify that the nested projections are merged when the subquery are eliminated - //@formatter:off - String query = """ - analyze - select x.col1, x.col2, x.col3 + x.col2 - from - ( - select col1, col2, col3 - from tableA - ) x - """; - //@formatter:on - - TestCatalog t = new TestCatalog(emptyMap()); - catalogRegistry.registerCatalog("t", t); - - QueryStatement queryStatement = parse(query); - queryStatement = StatementPlanner.plan(session, queryStatement); - - IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() - .get(0)).getPlan(); - - TableSourceReference tableA = new TableSourceReference(1, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), ""); - TableSourceReference subQueryX = new TableSourceReference(0, TableSourceReference.Type.SUBQUERY, "", QualifiedName.of("x"), "x"); - Schema expectedSchemaA = Schema.of(ast("", tableA)); - - //@formatter:off - IPhysicalPlan expected = - new DescribePlan( - 6, - new AnalyzeInterceptor( - 5, - new Projection( - 4, - new AnalyzeInterceptor( - 1, - new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()) - ), - Schema.of( - nast("col1", ResolvedType.ANY, tableA), - nast("col2", ResolvedType.ANY, tableA), - col("", Type.Any, "col3 + col2") - ), - List.of(cre("col1", tableA, CoreColumn.Type.NAMED_ASTERISK), cre("col2", tableA, CoreColumn.Type.NAMED_ASTERISK), - add(cre("col3", tableA, CoreColumn.Type.NAMED_ASTERISK), cre("col2", tableA, CoreColumn.Type.NAMED_ASTERISK))), - subQueryX - ) - ), - true); - //@formatter:on - - // System.out.println(actual.print(0)); - // System.out.println(expected.print(0)); - - Assertions.assertThat(actual) - .usingRecursiveComparison() - .isEqualTo(expected); - - assertEquals(expected, actual); - } - @Test void test_sub_query_projection_elimination_2() { @@ -1112,7 +989,7 @@ public TableSchema getTableSchema(IExecutionContext context, String catalogAlias }; catalogRegistry.registerCatalog("t", t); - session.setSystemProperty(QuerySession.FORCE_NO_INNER_CACHE, ValueVector.literalBoolean(true, 1)); + // session.setSystemProperty(QuerySession.FORCE_NO_INNER_CACHE, ValueVector.literalBoolean(true, 1)); QueryStatement queryStatement = parse(query); queryStatement = StatementPlanner.plan(session, queryStatement); @@ -2998,84 +2875,6 @@ void test_predicate_pushdown_2() assertEquals(expected, actual); } - @Test - void test_predicate_pushdown_with_analyze() - { - //@formatter:off - String query = "" - + "analyze select * " - + "from \"table\" " - + "where col = 10 " // Comparison - + "and col2 in (10,20) " // In - + "and col3 like 'string' " // Like - + "and col4 is null " // Null - + "and someFunc() "; // Function call - - //@formatter:on - - // Capture col and "" (someFunc) - ScalarFunctionInfo someFunc = new ScalarFunctionInfo("someFunc", FunctionInfo.FunctionType.SCALAR) - { - }; - TestCatalog catalog = new TestCatalog(ofEntries(entry(QualifiedName.of("table"), CollectionUtils.asSet("col", "")))) - { - { - registerFunction(someFunc); - } - }; - - catalogRegistry.registerCatalog("t", catalog); - - QueryStatement queryStatement = parse(query); - queryStatement = StatementPlanner.plan(session, queryStatement); - - IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() - .get(0)).getPlan(); - - TableSourceReference table = new TableSourceReference(0, TableSourceReference.Type.TABLE, "", QualifiedName.of("table"), ""); - - //@formatter:off - Schema expectedSchema = Schema.of(ast("", table)); - - IPhysicalPlan expected = new DescribePlan( - 4, - new AnalyzeInterceptor( - 3, - new Filter( - 2, - new AnalyzeInterceptor( - 1, - new TableScan(0, expectedSchema, table, "test", catalog.scanDataSources.get(0), emptyList()) - ), - new ExpressionPredicate( - and( - and( - new InExpression(cre("col2", table, CoreColumn.Type.NAMED_ASTERISK), asList(intLit(10), intLit(20)), false), - new LikeExpression(cre("col3", table, CoreColumn.Type.NAMED_ASTERISK), new LiteralStringExpression("string"), false, null) - ), - new NullPredicateExpression(cre("col4", table, CoreColumn.Type.NAMED_ASTERISK), false))) - )), - true); - //@formatter:on - - //@formatter:off - assertEquals(1, catalog.consumedPredicate.size()); - assertEquals(asList( - Triple.of(null, IPredicate.Type.FUNCTION_CALL, asList(new FunctionCallExpression("t", someFunc, null, asList()))), - Triple.of(QualifiedName.of("col"), IPredicate.Type.COMPARISION, asList(intLit(10))) - ), catalog.consumedPredicate.get(QualifiedName.of("table"))); - //@formatter:on - - // System.out.println(actual.print(0)); - // System.out.println(expected.print(0)); - - Assertions.assertThat(actual) - .usingRecursiveComparison() - .isEqualTo(expected); - - assertEquals(expected, actual); - } - @Test void test_predicate_pushdown_all_filters_consumed() { @@ -3126,7 +2925,7 @@ void test_predicate_pushdown_all_filters_consumed() assertEquals(expected, actual); } - private QueryStatement parse(String query) + static QueryStatement parse(String query) { return PARSER.parseQuery(query, null); } diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/test/TestHarnessRunner.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/test/TestHarnessRunner.java index 341a1f029..7f9efd43b 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/test/TestHarnessRunner.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/test/TestHarnessRunner.java @@ -389,8 +389,31 @@ private void assertResultSetEqual(boolean onlyAssertExpectedColumns, boolean sch { ColumnValue expectedColumn = j < expectedRow.size() ? expectedRow.get(j) : null; - ColumnValue actualColumn = j < actualRow.size() ? actualRow.get(j) - : null; + + ColumnValue actualColumn = null; + // Find the expected column among the actual + if (onlyAssertExpectedColumns) + { + for (int k = j; k < actualRow.size(); k++) + { + if (expectedColumn.getKey() + .equalsIgnoreCase(actualRow.get(k) + .getKey())) + { + actualColumn = actualRow.get(k); + } + } + if (actualColumn == null) + { + throw new IllegalArgumentException("No column found among actual with name: " + expectedColumn.getKey()); + } + } + else + { + actualColumn = j < actualRow.size() ? actualRow.get(j) + : null; + + } if (!equals(expectedColumn, actualColumn)) { diff --git a/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json b/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json index 34b518063..444c066bb 100644 --- a/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json +++ b/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json @@ -809,6 +809,7 @@ }, { "name": "Describe select, schema full, any vectors", + "onlyAssertExpectedColumns": true, "schemaLess": false, "typedVectors": false, "query": [ @@ -822,18 +823,19 @@ ], "expectedResultSets": [ [ - [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], - [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Hash Time", "value": "00:00:00.000" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Hash Time", "value": "00:00:00.000" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] ] ] }, { - "name": "Analyze select, schema full", - "ignore": true, - "_todo": "Add support for regex assert columns, this query contains times that isn't deterministic", - "schemaLess": false, + "name": "Analyze select, schema less, any vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": true, + "typedVectors": false, "query": [ "analyze ", "select d.id ", @@ -845,15 +847,129 @@ ], "expectedResultSets": [ [ - [{ "key": "Node Id", "value": 3 }, { "key": "Name", "value": "+- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 2 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], - [{ "key": "Node Id", "value": 1 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Hash Time", "value": "00:00:00.000" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 0 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + [{ "key": "Node Id", "value": 8 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": 8 }, { "key": "Name", "value": "| +- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "d (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + ] + ] + }, + { + "name": "Analyze select 2, schema full, any vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": false, + "query": [ + "analyze ", + "select * ", + "from data d", + "inner join source s ", + " on s.col1 = d.id " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 6 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 6 }, { "key": "Name", "value": "| +- Hash Match" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any), col1 (Any), col2 (Any)" }, { "key": "Catalog", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any)" }, { "key": "Catalog", "value": "Test#c" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: source" }, { "key": "Output", "value": "col1 (Any), col2 (Any)" }, { "key": "Catalog", "value": "Test#c" }] + ] + ] + }, + { + "name": "Analyze select 3, schema full, types vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": true, + "query": [ + "set @@force_nested_loop=true ", + "analyze ", + "select * ", + "from data d", + "inner join source s ", + " on s.col1 = d.id " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 6 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 6 }, { "key": "Name", "value": "| +- Nested Loop" }, { "key": "Output", "value": "id (Int), row_id (Int), data (String), col1 (Int), col2 (Int)" }, { "key": "Catalog", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Int), row_id (Int), data (String)" }, { "key": "Catalog", "value": "Test#c" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: source" }, { "key": "Output", "value": "col1 (Int), col2 (Int)" }, { "key": "Catalog", "value": "Test#c" }] + ] + ] + }, + { + "name": "Analyze select 4, schema full, types vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": true, + "query": [ + "analyze ", + "select top 1 * ", + "from (", + " values (1,2), (3,4) ", + ") x (col1, col2) " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": null } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Limit" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Constant Scan" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 2}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ] + ] + ] + }, + { + "name": "Analyze select 5, dynamic no extended output, schema full, types vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": true, + "query": [ + "set @@plan_analyze = true ", + "set @@plan_format = 'table' ", + "set @@plan_extended_output = false ", + + "", + "select top 1 * ", + "from (", + " values (1,2), (3,4) ", + ") x (col1, col2) " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": null } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Limit" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Constant Scan" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 2}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ] + ] + ] + }, + { + "name": "Analyze select 6, dynamic extended output, schema full, types vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": true, + "query": [ + "set @@plan_analyze = true ", + "set @@plan_format = 'table' ", + "", + "select top 1 * ", + "from (", + " values (1,2), (3,4) ", + ") x (col1, col2) " + ], + "expectedResultSets": [ + [ + [{ "key": "col1", "value": 1 }, { "key": "col2", "value": 2 } ] + ], + [ + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": null } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Limit" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Constant Scan" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 2}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ] ] ] }, { "name": "Describe select, schema less", + "onlyAssertExpectedColumns": true, "schemaLess": true, "query": [ "describe ", @@ -866,10 +982,36 @@ ], "expectedResultSets": [ [ - [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], - [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Hash Time", "value": "00:00:00.000" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "d (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "d (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + ] + ] + }, + { + "name": "Describe select, dynamic no extended output, schema less", + "onlyAssertExpectedColumns": true, + "schemaLess": true, + "query": [ + "set @@plan_describe = true ", + "set @@plan_extended_output = false ", + "", + "select d.id ", + ", max(row_id) ", + "from data d", + "group by d.id ", + "having count(1) = 1 ", + "order by id " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "d (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] ] ] }, diff --git a/payloadbuilder-core/src/test/resources/harnessCases/TemporaryTables.json b/payloadbuilder-core/src/test/resources/harnessCases/TemporaryTables.json index 446e27c89..983cc4d0d 100644 --- a/payloadbuilder-core/src/test/resources/harnessCases/TemporaryTables.json +++ b/payloadbuilder-core/src/test/resources/harnessCases/TemporaryTables.json @@ -110,9 +110,10 @@ [{ "key": "newCol1", "value": 10 }, { "key": "newCol2", "value": 60 } ] ], [ - [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Hash Match" } ], - [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" } ], - [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Index Seek: \"#\".temp [newCol1] (ALL)" } ] + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" } ], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Hash Match" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Index Seek: \"#\".temp [newCol1] (ALL)" } ] ], [ [{ "key": "id", "value": 2 }, { "key": "row_id", "value": 1 }, { "key": "data", "value": "2_1" }, { "key": "newCol1", "value": 2 }, { "key": "newCol2", "value": 20 } ] @@ -144,9 +145,10 @@ "expectedResultSets": [ [], [ - [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Hash Match" } ], - [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" } ], - [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Index Seek: \"#\".temp [newCol1] (ALL)" } ] + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" } ], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Hash Match" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Index Seek: \"#\".temp [newCol1] (ALL)" } ] ], [] ]