diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java index 5a462019..00f4e6ec 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java @@ -8,6 +8,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.time.StopWatch; import se.kuseman.payloadbuilder.api.OutputWriter; @@ -25,6 +26,11 @@ import se.kuseman.payloadbuilder.core.execution.OutputWriterUtils; import se.kuseman.payloadbuilder.core.execution.QuerySession; import se.kuseman.payloadbuilder.core.execution.StatementContext; +import se.kuseman.payloadbuilder.core.parser.Location; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeData; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnlayzeType; +import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan; import se.kuseman.payloadbuilder.core.physicalplan.IPhysicalPlan; import se.kuseman.payloadbuilder.core.statement.CacheFlushRemoveStatement; import se.kuseman.payloadbuilder.core.statement.DescribeSelectStatement; @@ -166,6 +172,27 @@ public Void visit(StatementList statement, Void context) public Void visit(PhysicalStatement statement, Void ctx) { currentPlan = statement.getPlan(); + + if (!(currentPlan instanceof DescribePlan)) + { + AnalyzeData analyzeData = AnalyzeVisitor.AnalyzeData.fromSession(session); + if (analyzeData.type() != AnlayzeType.NONE) + { + String queryText = ObjectUtils.getIfNull(statement.getLocation(), Location.EMPTY) + .text(); + IPhysicalPlan analyzePlan = AnalyzeVisitor.describe(currentPlan, analyzeData.type(), analyzeData.format(), queryText); + if (analyzeData.extendOutput()) + { + // Run currentPlan as usual and the add the analyze on top on queue + queue.add(0, new PhysicalStatement(analyzePlan, null)); + } + else + { + currentPlan = analyzePlan; + } + } + } + return null; } @@ -222,7 +249,7 @@ protected void processCurrentPlan(OutputWriter writer, ResultConsumer resultCons Schema schema = currentPlan.getSchema(); boolean currentPlanIsAsterisk = SchemaUtils.isAsterisk(schema); boolean currentPlanHasAsteriskInput = currentPlanIsAsterisk - || SchemaUtils.isAsterisk(schema, true); + || SchemaUtils.originatesFromAsteriskInput(schema); if (resultConsumer != null) { resultConsumer.schema(currentPlanIsAsterisk ? Schema.EMPTY diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/common/SchemaUtils.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/common/SchemaUtils.java index 592d2e27..ba41b169 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/common/SchemaUtils.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/common/SchemaUtils.java @@ -94,14 +94,22 @@ else if (schema2 == null return new Schema(columns); } - /** Returns true if this schema contains asterisk columns */ + /** + * Returns true if provided schema has columns that originates from any asterisk input. Ie. any column on any level that is of type {@link CoreColumn.Type#ASTERISK} or + * {@link CoreColumn.Type#NAMED_ASTERISK} + */ + public static boolean originatesFromAsteriskInput(Schema schema) + { + return isAsterisk(schema, true, true); + } + + /** Returns true if this schema contains asterisk columns. One or more columns of type {@link CoreColumn.Type#ASTERISK}. */ public static boolean isAsterisk(Schema schema) { - return isAsterisk(schema, false); + return isAsterisk(schema, false, false); } - /** Returns true if this schema contains asterisk columns */ - public static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks) + private static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks, boolean traverseIntoSubSchemas) { int size = schema.getSize(); if (size == 0) @@ -113,7 +121,7 @@ public static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks) { Column column = schema.getColumns() .get(i); - if (isAsterisk(column, includeNamedAsterisks)) + if (isAsterisk(column, includeNamedAsterisks, traverseIntoSubSchemas)) { return true; } @@ -126,13 +134,10 @@ public static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks) */ public static boolean isAsterisk(Column column) { - return isAsterisk(column, false); + return isAsterisk(column, false, false); } - /** - * Returns true if provided column is asterisk. - */ - public static boolean isAsterisk(Column column, boolean includeNamedAsterisks) + private static boolean isAsterisk(Column column, boolean includeNamedAsterisks, boolean traverseIntoSubSchemas) { if (column instanceof CoreColumn cc) { @@ -144,11 +149,15 @@ public static boolean isAsterisk(Column column, boolean includeNamedAsterisks) } // Dig down into complex type schema - if (cc.getType() - .getSchema() != null) + // We also do this for populated columns as those count as asterisk + // if their column is asterisk + if ((traverseIntoSubSchemas + || cc.getColumnType() == CoreColumn.Type.POPULATED) + && cc.getType() + .getSchema() != null) { return isAsterisk(cc.getType() - .getSchema(), includeNamedAsterisks); + .getSchema(), includeNamedAsterisks, traverseIntoSubSchemas); } } return false; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/execution/QuerySession.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/execution/QuerySession.java index ab65d37b..cb1386f6 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/execution/QuerySession.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/execution/QuerySession.java @@ -55,17 +55,26 @@ **/ public class QuerySession implements IQuerySession { - /* System properties */ + /* Compile time System properties */ /** Enable to print logical and physical plans to session print writer */ public static final String PRINT_PLAN = "printplan"; /** Enable to print all logical plans for each optimisation rule. */ public static final String DEBUG_PLAN = "debugplan"; - /** Force a nested loop where default would have been a hash match */ public static final String FORCE_NESTED_LOOP = "force_nested_loop"; + // ** Force no inner cache for non correlated nested loops */ + // public static final String FORCE_NO_INNER_CACHE = "force_no_inner_cache"; + + /* Runtime System properties */ + /** Enable describe of all plans without explicit DESCRIBE key word. This dynamically injects a describe plan on top of all plans. */ + public static final String PLAN_DESCRIBE = "plan_describe"; + /** Enable analyze of all plans without explicit ANALYZE key word. This dynamically injects a analyze plan on top of all plans. */ + public static final String PLAN_ANALYZE = "plan_analyze"; + /** Format of plan output. Only applicable {@link #PLAN_ANALYZE}, {@link #PLAN_DESCRIBE} or explicit DESCRIBE/ANALYZE keyword is set. */ + public static final String PLAN_FORMAT = "plan_format"; + /** Flag that indicates that the query result should be returned along with the plan. Only applicable {@link #PLAN_ANALYZE}, {@link #PLAN_DESCRIBE} or explicit DESCRIBE/ANALYZE keyword is set. */ + public static final String PLAN_EXTENDED_OUTPUT = "plan_extended_output"; - /** Force no inner cache for non correlated nested loops */ - public static final String FORCE_NO_INNER_CACHE = "force_no_inner_cache"; /* End system properties */ /* Compile fields */ diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolver.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolver.java index 82cf2d5b..6e080b34 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolver.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolver.java @@ -699,19 +699,9 @@ public ILogicalPlan visit(OperatorFunctionScan plan, Ctx context) ResolvedType type = pair.getValue() .getType(input.getSchema()); - // If the function type has a sub schema which is asterisk then we treat this column as asterisk - // to easier detect if a schema is asterisk without the need to dig into the sub schema - CoreColumn.Type columnType = CoreColumn.Type.REGULAR; - if (type.getSchema() != null - && SchemaUtils.isAsterisk(type.getSchema(), true)) - { - columnType = CoreColumn.Type.ASTERISK; - } - // Recreate the operator schema // Use type from the resolved input and name from the plans schema Schema schema = Schema.of(CoreColumn.Builder.from(name, type) - .withColumnType(columnType) .withInternal(isInternal) .build()); @@ -1372,15 +1362,6 @@ private ResolveResult resolve(ColumnResolver.Ctx context, QualifiedName original // We only look for direct asterisk columns here boolean isAsterisk = SchemaUtils.getColumnType(schemaColumn) == CoreColumn.Type.ASTERISK; - // We treat internal columns as non asterisk when resolving even if their schema is asterisk due - // to it's input. - // This happens when a subquery expression is pushed down to a join and is getting a generated name (__expr0 etc.) - // and we hit that column, if that one is asterisk we must treat it like a non asterisk match - if (SchemaUtils.isInternal(schemaColumn)) - { - isAsterisk = false; - } - String schemaAlias = p.getKey(); // If the schema doesn't have an alias check to see if the column has a TableSourceReference // attached and if so use the as schemaAlias diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/Location.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/Location.java index 9905bab7..fde80773 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/Location.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/Location.java @@ -1,18 +1,26 @@ package se.kuseman.payloadbuilder.core.parser; +import static java.util.Objects.requireNonNull; + import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.misc.Interval; import org.antlr.v4.runtime.tree.TerminalNode; /** A location in query text */ -public record Location(int line, int startOffset, int endOffset) +public record Location(int line, int startOffset, int endOffset, String text) { - public static Location EMPTY = new Location(0, 0, 0); + public static Location EMPTY = new Location(0, 0, 0, ""); + + public Location + { + requireNonNull(text); + } /** Create a location from provided rule context */ public static Location from(ParserRuleContext ctx) { - return new Location(ctx.start.getLine(), ctx.start.getStartIndex(), ctx.stop.getStopIndex() + 1); + return new Location(ctx.start.getLine(), ctx.start.getStartIndex(), ctx.stop.getStopIndex() + 1, ""); } /** Create a location from provided rule context */ @@ -23,6 +31,17 @@ public static Location from(TerminalNode node) node.getSymbol() .getStartIndex(), node.getSymbol() - .getStopIndex() + 1); + .getStopIndex() + 1, + ""); } + + /** Create a location from provided rule context extracting the rule text. */ + public static Location withText(ParserRuleContext ctx) + { + String text = ctx.start.getInputStream() + .getText(new Interval(ctx.start.getStartIndex(), ctx.stop.getStopIndex())); + + return new Location(ctx.start.getLine(), ctx.start.getStartIndex(), ctx.stop.getStopIndex() + 1, text); + } + } diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/QueryParser.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/QueryParser.java index e5bb0677..4d037d11 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/QueryParser.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/QueryParser.java @@ -228,7 +228,7 @@ else if (e instanceof LexerNoViableAltException) } } - Location location = new Location(line, startIndex, stopIndex); + Location location = new Location(line, startIndex, stopIndex, ""); throw new ParseException(msg, location); } }; @@ -469,10 +469,10 @@ public Object visitInsertStatement(InsertStatementContext ctx) inputInput = limit.getInput(); } - input = new LogicalSelectStatement(new Limit(inputInput, topCount), false); + input = new LogicalSelectStatement(new Limit(inputInput, topCount), false, Location.withText(ctx)); } TableName tableName = tableName(ctx.tableName(), ctx.intoOptions); - return new LogicalInsertIntoStatement(input, tableName.catalogAlias, tableName.table, columns, tableName.options, Location.from(ctx)); + return new LogicalInsertIntoStatement(input, tableName.catalogAlias, tableName.table, columns, tableName.options, Location.withText(ctx)); } @Override @@ -490,7 +490,7 @@ public Object visitInsertStatementValue(InsertStatementValueContext ctx) .toList(); ConstantScan constantScan = ConstantScan.createFromRows(expressions, Location.from(ctx.tableValueConstructor())); - return new LogicalSelectStatement(constantScan, false); + return new LogicalSelectStatement(constantScan, false, Location.withText(ctx)); } @Override @@ -531,7 +531,7 @@ public Object visitSelectStatement(SelectStatementContext ctx) plan = wrapTop(plan, ctx); plan = wrapOperatorFunction(plan, ctx); - Statement statement = new LogicalSelectStatement(plan, assignmentSelect); + Statement statement = new LogicalSelectStatement(plan, assignmentSelect, Location.withText(ctx)); if (ctx.into != null) { @@ -541,7 +541,7 @@ public Object visitSelectStatement(SelectStatementContext ctx) } TableName tableName = tableName(ctx.into, ctx.intoOptions); - statement = new LogicalSelectIntoStatement((LogicalSelectStatement) statement, tableName.catalogAlias, tableName.table, tableName.options, Location.from(ctx.into)); + statement = new LogicalSelectIntoStatement((LogicalSelectStatement) statement, tableName.catalogAlias, tableName.table, tableName.options, Location.withText(ctx)); } return statement; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanRewriter.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanRewriter.java new file mode 100644 index 00000000..cad53052 --- /dev/null +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanRewriter.java @@ -0,0 +1,141 @@ +package se.kuseman.payloadbuilder.core.physicalplan; + +import java.util.List; + +/** Base class for rewriters of {@link IPhysicalPlan}'s */ +public abstract class APhysicalPlanRewriter extends APhysicalPlanVisitor +{ + protected IPhysicalPlan visit(IPhysicalPlan plan, C context) + { + return plan.accept(this, context); + } + + @Override + public IPhysicalPlan visit(Projection plan, C context) + { + return new Projection(plan.getNodeId(), visit(plan.getInput(), context), plan.getSchema(), plan.getExpressions(), plan.getParentTableSource()); + } + + @Override + public IPhysicalPlan visit(Sort plan, C context) + { + return new Sort(plan.getNodeId(), visit(plan.getInput(), context), plan.getSortItems()); + } + + @Override + public IPhysicalPlan visit(Filter plan, C context) + { + return new Filter(plan.getNodeId(), visit(plan.getInput(), context), plan.getPredicate()); + } + + @Override + public IPhysicalPlan visit(HashAggregate plan, C context) + { + return new HashAggregate(plan.getNodeId(), visit(plan.getInput(), context), plan.getAggregateExpressions(), plan.getProjectionExpressions(), plan.getParentTableSource()); + } + + @Override + public IPhysicalPlan visit(TableScan plan, C context) + { + // Nothing to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(IndexSeek plan, C context) + { + // Nothing to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(TableFunctionScan plan, C context) + { + // Nothing to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(ExpressionScan plan, C context) + { + // Nothing to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(NestedLoop plan, C context) + { + return NestedLoop.copy(plan, visit(plan.getOuter(), context), visit(plan.getInner(), context)); + } + + @Override + public IPhysicalPlan visit(HashMatch plan, C context) + { + return new HashMatch(plan, visit(plan.getOuter(), context), visit(plan.getInner(), context)); + } + + @Override + public IPhysicalPlan visit(OperatorFunctionScan plan, C context) + { + return new OperatorFunctionScan(plan.getNodeId(), visit(plan.getInput(), context), plan.getFunction(), plan.getCatalogAlias(), plan.getSchema()); + } + + @Override + public IPhysicalPlan visit(ConstantScan plan, C context) + { + // Noting to rewrite + return plan; + } + + @Override + public IPhysicalPlan visit(Limit plan, C context) + { + return new Limit(plan.getNodeId(), visit(plan.getInput(), context), plan.getLimitExpression()); + } + + @Override + public IPhysicalPlan visit(Assert plan, C context) + { + return Assert.copy(plan, visit(plan.getInput(), context)); + } + + @Override + public IPhysicalPlan visit(Concatenation plan, C context) + { + List inputs = plan.getInputs() + .stream() + .map(p -> visit(p, context)) + .toList(); + return new Concatenation(plan.getNodeId(), plan.getSchema(), inputs); + } + + @Override + public IPhysicalPlan visit(AnalyzeInterceptor plan, C context) + { + return new AnalyzeInterceptor(plan.getNodeId(), visit(plan.getInput(), context)); + } + + @Override + public IPhysicalPlan visit(AssignmentPlan plan, C context) + { + return new AssignmentPlan(plan.getNodeId(), visit(plan.getInput(), context)); + } + + @Override + public IPhysicalPlan visit(CachePlan plan, C context) + { + return new CachePlan(plan.getNodeId(), visit(plan.getInput(), context)); + } + + @Override + public IPhysicalPlan visit(DescribePlan plan, C context) + { + return new DescribePlan(plan.getNodeId(), visit(plan.getInput(), context), plan.isAnalyze(), plan.getAnalyzeFormat(), plan.getQueryText()); + } + + @Override + public IPhysicalPlan visit(InsertInto plan, C context) + { + return new InsertInto(plan.getNodeId(), visit(plan.getInput(), context), plan.getInsertColumns(), plan.getDatasink()); + } +} diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanVisitor.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanVisitor.java new file mode 100644 index 00000000..9f8a4c5c --- /dev/null +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanVisitor.java @@ -0,0 +1,146 @@ +package se.kuseman.payloadbuilder.core.physicalplan; + +/** Base class for logical visitors */ +public abstract class APhysicalPlanVisitor implements IPhysicalPlanVisitor +{ + protected T defaultResult(C context) + { + return null; + } + + protected T aggregate(T result, T nextResult) + { + return nextResult; + } + + protected T visitChildren(C context, IPhysicalPlan plan) + { + T result = defaultResult(context); + for (IPhysicalPlan p : plan.getChildren()) + { + T value = p.accept(this, context); + result = aggregate(result, value); + } + return result; + } + + @Override + public T visit(Projection plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Sort plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Filter plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(HashAggregate plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(TableScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(TableFunctionScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(ExpressionScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(NestedLoop plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(HashMatch plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(OperatorFunctionScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(ConstantScan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Limit plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Assert plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(Concatenation plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(IndexSeek plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(AnalyzeInterceptor plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(AssignmentPlan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(CachePlan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(DescribePlan plan, C context) + { + return visitChildren(context, plan); + } + + @Override + public T visit(InsertInto plan, C context) + { + return visitChildren(context, plan); + } +} diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeInterceptor.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeInterceptor.java index 8c9efe1e..5d54d96c 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeInterceptor.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeInterceptor.java @@ -34,6 +34,12 @@ public IPhysicalPlan getInput() return input; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public List getChildNodes() { @@ -93,18 +99,9 @@ public TupleIterator execute(IExecutionContext context) final AnalyzeData data = context.getStatementContext() .getOrCreateNodeData(nodeId, AnalyzeData::new); data.increaseExecutionCount(); - - if (data.iterator == null) - { - data.iterator = new AnalyzeIterator(data); - } - data.resumeNodeTime(); - final TupleIterator it = input.execute(context); + data.iterator.it = input.execute(context); data.suspenNodeTime(); - - data.iterator.it = it; - return data.iterator; } @@ -241,6 +238,6 @@ public void close() /** Analyze data */ private static class AnalyzeData extends NodeData { - AnalyzeIterator iterator; + AnalyzeIterator iterator = new AnalyzeIterator(this); } } diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeVisitor.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeVisitor.java new file mode 100644 index 00000000..78eaa817 --- /dev/null +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AnalyzeVisitor.java @@ -0,0 +1,147 @@ +package se.kuseman.payloadbuilder.core.physicalplan; + +import java.util.ArrayList; +import java.util.List; + +import se.kuseman.payloadbuilder.api.execution.ValueVector; +import se.kuseman.payloadbuilder.core.execution.QuerySession; + +/** Visitor that creates a analyze plan from input plan. */ +public class AnalyzeVisitor extends APhysicalPlanRewriter +{ + private static final AnalyzeVisitor INSTANCE = new AnalyzeVisitor(); + + static class Context + { + int nodeId; + } + + @Override + protected IPhysicalPlan visit(IPhysicalPlan plan, Context context) + { + // Make sure we don't produce analyze plans of already analyzed plans + if (plan instanceof AnalyzeInterceptor) + { + return plan; + } + + IPhysicalPlan analyzePlan = super.visit(plan, context); + return new AnalyzeInterceptor(context.nodeId++, analyzePlan); + } + + /** Describe provided plan with analyze type. */ + public static IPhysicalPlan describe(IPhysicalPlan plan, AnlayzeType analyzeType, AnalyzeFormat analyzeFormat, String queryText) + { + if (plan instanceof DescribePlan) + { + return plan; + } + + boolean isAnalyze = analyzeType == AnlayzeType.ANALYZE; + int nodeId = -1; + if (isAnalyze) + { + plan = analyze(plan); + nodeId = plan.getActualNodeId() + 1; + } + else + { + nodeId = getMaxNodeId(plan); + } + + return new DescribePlan(nodeId, plan, isAnalyze, analyzeFormat, queryText); + } + + /** Create analyze plan from input plan. */ + private static IPhysicalPlan analyze(IPhysicalPlan plan) + { + int maxNodeId = getMaxNodeId(plan); + Context ctx = new Context(); + ctx.nodeId = maxNodeId; + return INSTANCE.visit(plan, ctx); + } + + private static int getMaxNodeId(IPhysicalPlan plan) + { + int maxNodeId = -1; + List queue = new ArrayList<>(); + queue.add(plan); + while (!queue.isEmpty()) + { + IPhysicalPlan current = queue.remove(0); + maxNodeId = Math.max(maxNodeId, current.getNodeId()); + queue.addAll(current.getChildren()); + } + maxNodeId++; + return maxNodeId; + } + + /** Data for performing an analzye of a query statement. */ + public record AnalyzeData(AnlayzeType type, AnalyzeFormat format, boolean extendOutput) + { + /** Extracts analyze data from a session instance. */ + public static AnalyzeData fromSession(QuerySession session) + { + return extract(null, session); + } + + /** Extracts analyze data from a session instance. */ + private static AnalyzeData extract(AnlayzeType type, QuerySession session) + { + boolean extendedOutput = true; + ValueVector prop = session.getSystemProperty(QuerySession.PLAN_EXTENDED_OUTPUT); + if (!prop.isNull(0)) + { + extendedOutput = prop.getBoolean(0); + } + + if (type == null) + { + type = AnlayzeType.NONE; + prop = session.getSystemProperty(QuerySession.PLAN_ANALYZE); + if (prop.getPredicateBoolean(0)) + { + type = AnlayzeType.ANALYZE; + } + if (type == AnlayzeType.NONE) + { + prop = session.getSystemProperty(QuerySession.PLAN_DESCRIBE); + if (prop.getPredicateBoolean(0)) + { + type = AnlayzeType.DESCRIBE; + } + } + } + + AnalyzeFormat format = AnalyzeFormat.TABLE; + prop = session.getSystemProperty(QuerySession.PLAN_FORMAT); + if (!prop.isNull(0)) + { + format = AnalyzeFormat.valueOf(prop.getString(0) + .toString() + .toUpperCase()); + } + + return new AnalyzeData(type, format, extendedOutput); + } + } + + /** Type of analyze. */ + public enum AnlayzeType + { + NONE, + /** Describe plan without execution. */ + DESCRIBE, + /** Describe plan with execution stats. */ + ANALYZE, + } + + /** Output format of analyze. */ + public enum AnalyzeFormat + { + /** JSON tree of plan. Produces a result set with one row and one column. */ + JSON, + /** Regular table vector. */ + TABLE + } +} diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Assert.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Assert.java index ce6e2bbd..f2c46b0c 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Assert.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Assert.java @@ -38,12 +38,28 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public Supplier> getPredicateSupplier() + { + return predicateSupplier; + } + @Override public String getName() { return "Assert"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { @@ -138,6 +154,12 @@ public String toString() return "Assert (" + nodeId + "), type: " + predicateSupplier; } + /** Copy assert with new input. */ + public static Assert copy(Assert source, IPhysicalPlan input) + { + return new Assert(source.nodeId, input, source.predicateSupplier); + } + /** Create an assert node that assert max rows from input */ public static Assert maxRowCount(int nodeId, IPhysicalPlan input, int maxRowCount) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AssignmentPlan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AssignmentPlan.java index 64588472..e260118d 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AssignmentPlan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/AssignmentPlan.java @@ -36,12 +36,23 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + @Override public String getName() { return input.getName(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/CachePlan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/CachePlan.java index 789867bc..f8270683 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/CachePlan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/CachePlan.java @@ -33,6 +33,11 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + @Override public Schema getSchema() { @@ -45,6 +50,12 @@ public String getName() return "Cache"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public TupleIterator execute(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Concatenation.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Concatenation.java index 587a76e8..849169cb 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Concatenation.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Concatenation.java @@ -39,6 +39,23 @@ public int getNodeId() return nodeId; } + public List getInputs() + { + return inputs; + } + + @Override + public String getName() + { + return "Concatenation"; + } + + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java index ef4b7d68..ced3ef6d 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ConstantScan.java @@ -12,6 +12,7 @@ import java.util.stream.IntStream; import se.kuseman.payloadbuilder.api.catalog.Column; +import se.kuseman.payloadbuilder.api.catalog.IDatasource; import se.kuseman.payloadbuilder.api.catalog.Schema; import se.kuseman.payloadbuilder.api.execution.IExecutionContext; import se.kuseman.payloadbuilder.api.execution.TupleIterator; @@ -38,7 +39,7 @@ public ConstantScan(int nodeId, Schema schema, List> rowsExpre this.schema = schema; this.rowsExpressions = requireNonNull(rowsExpressions); this.vector = null; - this.hasAsteriskSchemaOrInput = SchemaUtils.isAsterisk(schema, true); + this.hasAsteriskSchemaOrInput = SchemaUtils.originatesFromAsteriskInput(schema); } public ConstantScan(int nodeId, TupleVector vector) @@ -62,6 +63,12 @@ public String getName() return "Constant Scan"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { @@ -70,6 +77,7 @@ public Map getDescribeProperties(IExecutionContext context) { if (!Schema.EMPTY.equals(vector.getSchema())) { + properties.put(IDatasource.OUTPUT, DescribeUtils.getOutputColumns(vector.getSchema())); properties.put("Rows", IntStream.range(0, vector.getRowCount()) .mapToObj(i -> "Row " + i + ": " diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribePlan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribePlan.java index e4e840a1..0045f9aa 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribePlan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribePlan.java @@ -23,6 +23,7 @@ import se.kuseman.payloadbuilder.core.execution.OutputWriterUtils; import se.kuseman.payloadbuilder.core.execution.QuerySession; import se.kuseman.payloadbuilder.core.execution.StatementContext; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeFormat; /** Plan for describing another physical plan and return describable output */ public class DescribePlan implements IPhysicalPlan @@ -31,12 +32,16 @@ public class DescribePlan implements IPhysicalPlan private final int nodeId; private final IPhysicalPlan input; private final boolean analyze; + private final String queryText; + private final AnalyzeFormat format; - public DescribePlan(int nodeId, IPhysicalPlan input, boolean analyze) + public DescribePlan(int nodeId, IPhysicalPlan input, boolean analyze, AnalyzeFormat format, String queryText) { this.nodeId = nodeId; this.input = requireNonNull(input, "input"); this.analyze = analyze; + this.format = requireNonNull(format, "format"); + this.queryText = queryText; } @Override @@ -45,6 +50,26 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public boolean isAnalyze() + { + return analyze; + } + + public AnalyzeFormat getAnalyzeFormat() + { + return format; + } + + public String getQueryText() + { + return queryText; + } + @Override public Schema getSchema() { @@ -57,47 +82,54 @@ public String getName() return "Write Output"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { Map properties = new LinkedHashMap<>(); - NodeData data = context.getStatementContext() - .getNodeData(nodeId); - AnalyzeInterceptor.populateTimings(context, data, this, properties); - properties.put("Allocations", ((ExecutionContext) context).getBufferAllocator() - .getStatistics() - .asObject()); - - Map> catalogStatistics = new HashMap<>(); - QuerySession session = (QuerySession) context.getSession(); - session.getCatalogRegistry() - .getCatalogs() - .forEach(e -> - { - Map statistics = e.getValue() - .getExecutionStatistics(context); - if (statistics.isEmpty()) - { - return; - } - catalogStatistics.put(e.getValue() - .getName(), statistics); - }); + properties.put("Query Text", queryText); - properties.put("Catalog Statistics", catalogStatistics); + if (analyze) + { + NodeData data = context.getStatementContext() + .getNodeData(nodeId); + AnalyzeInterceptor.populateTimings(context, data, this, properties); + properties.put("Allocations", ((ExecutionContext) context).getBufferAllocator() + .getStatistics() + .asObject()); + + Map> catalogStatistics = new HashMap<>(); + QuerySession session = (QuerySession) context.getSession(); + session.getCatalogRegistry() + .getCatalogs() + .forEach(e -> + { + Map statistics = e.getValue() + .getExecutionStatistics(context); + if (statistics.isEmpty()) + { + return; + } + catalogStatistics.put(e.getValue() + .getName(), statistics); + }); + + properties.put("Catalog Statistics", catalogStatistics); + } return properties; } @Override public TupleIterator execute(IExecutionContext context) { - IPhysicalPlan describePlan = input; - // Execute and traverse query in analyze mode before gathering describe data if (analyze) { - describePlan = this; - // Trigger creation of node data to get this nodes timing in output NodeData data = context.getStatementContext() .getOrCreateNodeData(nodeId); @@ -118,20 +150,20 @@ public TupleIterator execute(IExecutionContext context) } finally { + it.close(); + data.suspenNodeTime(); data.increaseRowCount(rowCount); // Populate total query time on every node data long total = TimeUnit.MILLISECONDS.convert(System.nanoTime() - start, TimeUnit.NANOSECONDS); ((StatementContext) context.getStatementContext()).forEachNodeData((i, n) -> n.setTotalQueryTime(total)); - - it.close(); } } ((ExecutionContext) context).getStatementContext() .setOuterTupleVector(new DescribeTupleVector()); - return TupleIterator.singleton(DescribeUtils.getDescribeVector(context, describePlan)); + return TupleIterator.singleton(DescribeUtils.getDescribeVector(context, format, this)); } @Override @@ -163,9 +195,8 @@ else if (obj == this) { return true; } - else if (obj instanceof DescribePlan) + else if (obj instanceof DescribePlan that) { - DescribePlan that = (DescribePlan) obj; return nodeId == that.nodeId && input.equals(that.input) && analyze == that.analyze; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribeUtils.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribeUtils.java index 75b5e19d..c1164582 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribeUtils.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/DescribeUtils.java @@ -8,9 +8,18 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.commons.lang3.mutable.MutableInt; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.introspect.VisibilityChecker; +import com.fasterxml.jackson.databind.json.JsonMapper; + import se.kuseman.payloadbuilder.api.catalog.Column; import se.kuseman.payloadbuilder.api.catalog.Column.Type; import se.kuseman.payloadbuilder.api.catalog.ResolvedType; @@ -20,10 +29,19 @@ import se.kuseman.payloadbuilder.api.execution.ValueVector; import se.kuseman.payloadbuilder.core.catalog.CoreColumn; import se.kuseman.payloadbuilder.core.common.DescribableNode; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeFormat; /** Utility class for building a describe result set */ public class DescribeUtils { + private static final String PLB_PLAN_JSON = "PLBPlanJson"; + + private static final JsonMapper MAPPER = JsonMapper.builder() + .visibility(VisibilityChecker.Std.defaultInstance() + .with(Visibility.NONE) + .withFieldVisibility(Visibility.ANY)) + .build(); + public static final String LOGICAL_OPERATOR = "Logical Operator"; public static final String POPULATING = "Populating"; public static final String BATCH_SIZE = "Batch Size"; @@ -65,10 +83,10 @@ public static String getOutputColumns(Schema schema) } /** Generate a describe tuple vector from provided plan */ - static TupleVector getDescribeVector(IExecutionContext context, DescribableNode node) + static TupleVector getDescribeVector(IExecutionContext context, AnalyzeFormat format, DescribableNode node) { final List describeRows = new ArrayList<>(); - collectDescribableRows(context, describeRows, -1, node, 0, "", false); + collectDescribableRows(context, format != AnalyzeFormat.JSON, describeRows, -1, node, 0, "", false); List describeColumns = new ArrayList<>(); Map countByColumn = new HashMap<>(); @@ -106,14 +124,8 @@ static TupleVector getDescribeVector(IExecutionContext context, DescribableNode for (DescribableRow row : describeRows) { Object[] values = new Object[size]; - if (row.nodeId >= 0) - { - values[0] = row.nodeId; - } - if (row.parentNodeId >= 0) - { - values[1] = row.parentNodeId; - } + values[0] = row.nodeId; + values[1] = row.parentNodeId; values[2] = row.name; for (int i = 3; i < size; i++) @@ -129,7 +141,7 @@ static TupleVector getDescribeVector(IExecutionContext context, DescribableNode .map(c -> new Column(c, ResolvedType.of(Type.Any))) .collect(toList())); - return new TupleVector() + TupleVector vector = new TupleVector() { @Override public Schema getSchema() @@ -148,7 +160,6 @@ public ValueVector getColumn(int column) { return new ValueVector() { - @Override public ResolvedType type() { @@ -164,7 +175,16 @@ public int size() @Override public boolean isNull(int row) { - return rows.get(row)[column] == null; + Object value = rows.get(row)[column]; + + if ((column == 0 + || column == 1) + && ((Integer) value) < 0) + { + value = null; + } + + return value == null; } @Override @@ -176,11 +196,56 @@ public Object getAny(int row) }; } }; + + if (format == AnalyzeFormat.TABLE) + { + return vector; + } + else if (format == AnalyzeFormat.JSON) + { + String json = getJson(describeRows); + return TupleVector.of(Schema.of(Column.of(PLB_PLAN_JSON, Type.String)), ValueVector.literalString(json, 1)); + } + + throw new IllegalArgumentException("Unsupported analyze format: " + format); + } + + private static String getJson(List rows) + { + Map rowByNodeId = rows.stream() + .collect(Collectors.toMap(r -> r.nodeId, Function.identity())); + + for (DescribableRow row : rows) + { + DescribableRow parent = rowByNodeId.get(row.parentNodeId); + if (parent == null) + { + continue; + } + else if (parent.children == null) + { + parent.children = new ArrayList<>(); + } + parent.children.add(row); + } + + try + { + return MAPPER.writerWithDefaultPrettyPrinter() + .writeValueAsString(rows.get(0)); + } + catch (JsonProcessingException e) + { + throw new RuntimeException("Error generating query plan as JSON", e); + } } - private static void collectDescribableRows(IExecutionContext context, List rows, int parentNodeId, DescribableNode parent, int pos, String indent, boolean last) + private static void collectDescribableRows(IExecutionContext context, boolean buildTextTree, List rows, int parentNodeId, DescribableNode parent, int pos, String indent, + boolean last) { - rows.add(new DescribableRow(parent.getNodeId(), parentNodeId, indent + "+- " + parent.getName(), parent.getDescribeProperties(context))); + String name = (buildTextTree ? (indent + "+- ") + : "") + parent.getName(); + rows.add(new DescribableRow(parent.getNodeId(), parentNodeId, name, parent.getDescribeProperties(context))); String nextIndent = indent + (last ? " " : "| "); int size = parent.getChildNodes() @@ -189,7 +254,7 @@ private static void collectDescribableRows(IExecutionContext context, List properties; + @JsonProperty + List children; DescribableRow(int nodeId, int parentNodeId, String name, Map properties) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ExpressionScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ExpressionScan.java index 6b41385e..b117d9cc 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ExpressionScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/ExpressionScan.java @@ -41,6 +41,18 @@ public int getNodeId() return nodeId; } + @Override + public String getName() + { + return "Expression Scan"; + } + + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Filter.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Filter.java index 31ce18a1..479cc725 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Filter.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Filter.java @@ -51,6 +51,12 @@ public String getName() return "Filter"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + public BiFunction getPredicate() { return predicate; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashAggregate.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashAggregate.java index 321e63b0..60824735 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashAggregate.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashAggregate.java @@ -87,6 +87,26 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public List getAggregateExpressions() + { + return aggregateExpressions; + } + + public List getProjectionExpressions() + { + return projectionExpressions; + } + + public TableSourceReference getParentTableSource() + { + return parentTableSource; + } + @Override public String getName() { @@ -94,6 +114,12 @@ public String getName() : "Hash Aggregate"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashMatch.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashMatch.java index b409ed93..201e77d5 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashMatch.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/HashMatch.java @@ -106,18 +106,40 @@ public HashMatch( this.isAsteriskInnerSchema = SchemaUtils.isAsterisk(inner.getSchema()); } + /** Copy ctor with new inputs */ + public HashMatch(HashMatch source, IPhysicalPlan outer, IPhysicalPlan inner) + { + this(source.nodeId, outer, inner, source.outerHashFunction, source.innerHashFunction, source.condition, source.populateAlias, source.emitEmptyOuterRows, source.pushOuterReference); + } + @Override public int getNodeId() { return nodeId; } + public IPhysicalPlan getOuter() + { + return outer; + } + + public IPhysicalPlan getInner() + { + return inner; + } + @Override public String getName() { return "Hash Match"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlan.java index 3dc9bc54..97fc92bd 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlan.java @@ -33,6 +33,9 @@ public interface IPhysicalPlan extends DescribableNode /** Return this plans children */ List getChildren(); + /** Accept visitor */ + T accept(IPhysicalPlanVisitor visitor, C context); + /** Returns true if this plan has any writable output else false */ default boolean hasWritableOutput() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlanVisitor.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlanVisitor.java new file mode 100644 index 00000000..c1ce8f0d --- /dev/null +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IPhysicalPlanVisitor.java @@ -0,0 +1,28 @@ +package se.kuseman.payloadbuilder.core.physicalplan; + +/** Visitor definition for {@link IPhysicalPlan}'s */ +public interface IPhysicalPlanVisitor +{ + //@formatter:off + T visit(Projection plan, C context); + T visit(Sort plan, C context); + T visit(Filter plan, C context); + T visit(HashAggregate plan, C context); + T visit(TableScan plan, C context); + T visit(IndexSeek plan, C context); + T visit(TableFunctionScan plan, C context); + T visit(ExpressionScan plan, C context); + T visit(NestedLoop plan, C context); + T visit(HashMatch plan, C context); + T visit(OperatorFunctionScan plan, C context); + T visit(ConstantScan plan, C context); + T visit(Limit plan, C context); + T visit(Assert plan, C context); + T visit(Concatenation plan, C context); + T visit(AnalyzeInterceptor plan, C context); + T visit(AssignmentPlan plan, C context); + T visit(CachePlan plan, C context); + T visit(DescribePlan plan, C context); + T visit(InsertInto plan, C context); + //@formatter:on +} diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IndexSeek.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IndexSeek.java index ccd65665..7bd1de18 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IndexSeek.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/IndexSeek.java @@ -38,6 +38,12 @@ public String getName() return "Index Seek: " + seekPredicate.getIndex(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java index 603da4eb..bc8a72ca 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/InsertInto.java @@ -30,7 +30,7 @@ public InsertInto(int nodeId, IPhysicalPlan input, List insertColumns, I this.input = requireNonNull(input, "input"); this.insertColumns = insertColumns; this.datasink = requireNonNull(datasink, " datasink"); - this.hasAsteriskSchemaOrInput = SchemaUtils.isAsterisk(input.getSchema(), true); + this.hasAsteriskSchemaOrInput = SchemaUtils.originatesFromAsteriskInput(input.getSchema()); } @Override @@ -39,12 +39,33 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public List getInsertColumns() + { + return insertColumns; + } + + public IDatasink getDatasink() + { + return datasink; + } + @Override public String getName() { return "Insert Into"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Limit.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Limit.java index 565d0f4c..d667b0b1 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Limit.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Limit.java @@ -39,6 +39,16 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public IExpression getLimitExpression() + { + return limitExpression; + } + @Override public Schema getSchema() { @@ -51,6 +61,12 @@ public String getName() return "Limit"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java index c54e8535..7e1f8158 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/NestedLoop.java @@ -93,12 +93,14 @@ public class NestedLoop implements IPhysicalPlan private final boolean switchedInputs; /** The outer schema if this join is a correlated type */ private final Schema outerSchema; - private final boolean isAsteriskOuterSchema; private final Schema schema; private final Schema cartesianSchema; private final boolean isAsteriskSchema; + private final boolean outerSchemaOriginatesFromAsteriskInput; + private final boolean schemaOriginatesFromAsteriskInput; + //@formatter:off private NestedLoop( int nodeId, @@ -124,7 +126,7 @@ private NestedLoop( this.emitEmptyOuterRows = emitEmptyOuterRows; this.switchedInputs = switchedInputs; this.outerSchema = requireNonNull(outerSchema, "outerSchema"); - this.isAsteriskOuterSchema = SchemaUtils.isAsterisk(outerSchema); + this.outerSchemaOriginatesFromAsteriskInput = SchemaUtils.originatesFromAsteriskInput(outerSchema); if (switchedInputs && (condition != null @@ -143,6 +145,14 @@ else if (pushOuterReference this.schema = getSchema(); this.cartesianSchema = getSchema(true); this.isAsteriskSchema = SchemaUtils.isAsterisk(schema); + this.schemaOriginatesFromAsteriskInput = SchemaUtils.originatesFromAsteriskInput(schema); + } + + /** Copy source with new inputs */ + public static NestedLoop copy(NestedLoop source, IPhysicalPlan outer, IPhysicalPlan inner) + { + return new NestedLoop(source.nodeId, outer, inner, source.condition, source.populateAlias, source.outerReferences, source.emitEmptyOuterRows, source.switchedInputs, source.pushOuterReference, + source.outerSchema); } /** Create an inner join. Logical operations: INNER JOIN */ @@ -223,12 +233,28 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getOuter() + { + return outer; + } + + public IPhysicalPlan getInner() + { + return inner; + } + @Override public String getName() { return "Nested Loop"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { @@ -318,7 +344,7 @@ else if (populateAlias != null) private TupleVector next; /** Bit set to keep track of outer matches. Used in left joins to know what outer indices to return */ private BitSet outerMatches; - private CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, isAsteriskSchema); + private CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, schemaOriginatesFromAsteriskInput); // The inner schema used when emitting empty outer row, will be the plan schema from start // but if there are inner matches before the un matched ones we switch @@ -593,7 +619,7 @@ private class PopulatingTupleIterator implements TupleIterator private TupleVector next; /** Bit set to keep track of outer matches. Used in left joins to know what outer indices to return */ private BitSet outerMatches; - private CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, isAsteriskSchema); + private CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, schemaOriginatesFromAsteriskInput); // The inner schema used when emitting empty outer row, will be the plan schema from start // but if there are inner matches before the un matched ones we switch @@ -800,7 +826,7 @@ private class LoopTupleIterator implements TupleIterator */ private final OuterTupleVector outerTupleVector; private final RowTupleVector rowTupleVector = new RowTupleVector(); - private final CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, isAsteriskSchema); + private final CartesianTupleVector cartesian = new CartesianTupleVector(cartesianSchema, schemaOriginatesFromAsteriskInput); private TupleVector currentOuter; private TupleVector next; @@ -816,7 +842,7 @@ private class LoopTupleIterator implements TupleIterator this.context = context; this.iterator = iterator; this.nodeData = nodeData; - this.outerTupleVector = new OuterTupleVector(contextOuterTupleVector, outerSchema, isAsteriskOuterSchema); + this.outerTupleVector = new OuterTupleVector(contextOuterTupleVector, outerSchema, outerSchemaOriginatesFromAsteriskInput); } @Override @@ -1091,7 +1117,7 @@ private TupleVector createUnmatchedOuterTuple(ExecutionContext context, Schema i s = SchemaUtils.joinSchema(outer.getSchema(), innerSchema, populateAlias); } - Schema schema = s; + final Schema schema = s; final int outerSize = outer.getSchema() .getSize(); return new TupleVector() @@ -1207,33 +1233,7 @@ void init(TupleVector outer) // We only assert that they are equal if (!outerSchemaIsAsterisk) { - assert (planOuterSchema.equals(this.schema)) : "Planned outer schema should match actual outer schema. Planned: " + planOuterSchema.getColumns() - .stream() - .map(o -> - { - String str = o.toString(); - TableSourceReference tableSource = SchemaUtils.getTableSource(o); - if (tableSource != null) - { - str += " (" + tableSource + ")"; - } - return str; - }) - .collect(joining(", ")) - + ", actual: " - + this.schema.getColumns() - .stream() - .map(o -> - { - String str = o.toString(); - TableSourceReference tableSource = SchemaUtils.getTableSource(o); - if (tableSource != null) - { - str += " (" + tableSource + ")"; - } - return str; - }) - .collect(joining(", ")); + CartesianTupleVector.assertSchemas(planOuterSchema, schema, "outer"); } } @@ -1267,7 +1267,7 @@ public ValueVector getColumn(int column) private static class CartesianTupleVector implements TupleVector { private final Schema cartesianSchema; - private final boolean isAsteriskSchema; + private final boolean schemaOriginatesFromAsteriskInput; private TupleVector outer; private TupleVector inner; @@ -1279,13 +1279,12 @@ private static class CartesianTupleVector implements TupleVector private CartesianColumn[] columns; private int rowCount; private int outerSize; - // private int innerSize; private int innerRowCount; - CartesianTupleVector(Schema cartesianSchema, boolean isAsteriskSchema) + CartesianTupleVector(Schema cartesianSchema, boolean schemaOriginatesFromAsteriskInput) { this.cartesianSchema = cartesianSchema; - this.isAsteriskSchema = isAsteriskSchema; + this.schemaOriginatesFromAsteriskInput = schemaOriginatesFromAsteriskInput; } /** Inits this vector with new outer/inner vectors. Calculates new schemas etc. */ @@ -1306,35 +1305,9 @@ void init(TupleVector outer, TupleVector inner) } // We only assert that they are equal - if (!isAsteriskSchema) + if (!schemaOriginatesFromAsteriskInput) { - assert (cartesianSchema.equals(this.schema)) : "Planned cartesian schema should match actual schema. Planned: " + cartesianSchema.getColumns() - .stream() - .map(o -> - { - String str = o.toString(); - TableSourceReference tableSource = SchemaUtils.getTableSource(o); - if (tableSource != null) - { - str += " (" + tableSource + ")"; - } - return str; - }) - .collect(joining(", ")) - + ", actual: " - + this.schema.getColumns() - .stream() - .map(o -> - { - String str = o.toString(); - TableSourceReference tableSource = SchemaUtils.getTableSource(o); - if (tableSource != null) - { - str += " (" + tableSource + ")"; - } - return str; - }) - .collect(joining(", ")); + assertSchemas(cartesianSchema, schema, "cartesian"); } this.outerSize = outerSchema.getSize(); @@ -1418,6 +1391,40 @@ protected int getRow(int row) return row % innerRowCount; } } + + // Validates that schemas are equal. Used in assert to catch errors in tests + static void assertSchemas(Schema plannedSchema, Schema runtimeSchema, String type) + { + assert (plannedSchema.equals(runtimeSchema)) : "Planned " + type + + " schema should match actual schema. Planned: " + + plannedSchema.getColumns() + .stream() + .map(o -> + { + String str = o.toString(); + TableSourceReference tableSource = SchemaUtils.getTableSource(o); + if (tableSource != null) + { + str += " (" + tableSource + ")"; + } + return str; + }) + .collect(joining(", ")) + + ", actual: " + + runtimeSchema.getColumns() + .stream() + .map(o -> + { + String str = o.toString(); + TableSourceReference tableSource = SchemaUtils.getTableSource(o); + if (tableSource != null) + { + str += " (" + tableSource + ")"; + } + return str; + }) + .collect(joining(", ")); + } } /** Tuple vector that wraps another tuple vector for a single row */ diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java index b18fd345..ed348983 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/OperatorFunctionScan.java @@ -35,7 +35,7 @@ public OperatorFunctionScan(int nodeId, IPhysicalPlan input, OperatorFunctionInf this.function = requireNonNull(function, "function"); this.catalogAlias = requireNonNull(catalogAlias, "catalogAlias"); this.schema = requireNonNull(schema, "schema"); - this.hasAsteriskSchemaOrInput = SchemaUtils.isAsterisk(schema, true); + this.hasAsteriskSchemaOrInput = SchemaUtils.originatesFromAsteriskInput(schema); if (schema.getColumns() .size() != 1) { @@ -49,6 +49,33 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public OperatorFunctionInfo getFunction() + { + return function; + } + + public String getCatalogAlias() + { + return catalogAlias; + } + + @Override + public String getName() + { + return "Operator Function Scan"; + } + + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Schema getSchema() { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java index b6ea1808..b0bc01d6 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Projection.java @@ -45,7 +45,7 @@ public Projection(int nodeId, IPhysicalPlan input, Schema schema, List e instanceof AsteriskExpression); this.hasAsteriskSchemaOrInput = hasAsteriskProjection - || SchemaUtils.isAsterisk(schema, true); + || SchemaUtils.originatesFromAsteriskInput(schema); this.parentTableSource = parentTableSource; if (expressions.size() != schema.getSize()) @@ -81,6 +81,12 @@ public String getName() return "Project"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Sort.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Sort.java index c430eeb3..2fab1ad9 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Sort.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/Sort.java @@ -49,12 +49,28 @@ public int getNodeId() return nodeId; } + public IPhysicalPlan getInput() + { + return input; + } + + public List getSortItems() + { + return sortItems; + } + @Override public String getName() { return "Sort"; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableFunctionScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableFunctionScan.java index c061de0c..dac08bd2 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableFunctionScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableFunctionScan.java @@ -63,6 +63,12 @@ public String getName() return "Function Scan: " + tableSource.getName(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java index 5071df61..1b4da216 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/physicalplan/TableScan.java @@ -59,6 +59,12 @@ public String getName() return "Scan: " + tableSource.getName(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + return visitor.visit(this, context); + } + @Override public Map getDescribeProperties(IExecutionContext context) { @@ -212,13 +218,28 @@ static boolean schemaEqualsRegardingTypeAndName(Schema expected, Schema actual) Column vectorColumn = actual.getColumns() .get(i); - if (!schemaColumn.getType() - .equals(vectorColumn.getType()) + if ((schemaColumn.getType() + .getType() != vectorColumn.getType() + .getType()) || !schemaColumn.getName() .equals(vectorColumn.getName())) { return false; } + + Schema schemaSchema = schemaColumn.getType() + .getSchema(); + if (schemaSchema == null) + { + continue; + } + + // Dig down into nested schemas + if (!schemaEqualsRegardingTypeAndName(schemaSchema, vectorColumn.getType() + .getSchema())) + { + return false; + } } } return true; diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java index dde11e90..dd5317dc 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/QueryPlanner.java @@ -63,8 +63,6 @@ import se.kuseman.payloadbuilder.core.logicalplan.TableScan; import se.kuseman.payloadbuilder.core.logicalplan.TableSource; import se.kuseman.payloadbuilder.core.logicalplan.optimization.ProjectionMerger; -import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeInterceptor; -import se.kuseman.payloadbuilder.core.physicalplan.CachePlan; import se.kuseman.payloadbuilder.core.physicalplan.ExpressionPredicate; import se.kuseman.payloadbuilder.core.physicalplan.HashAggregate; import se.kuseman.payloadbuilder.core.physicalplan.HashMatch; @@ -102,15 +100,8 @@ public IPhysicalPlan visit(Projection plan, Context context) input = p.getInput(); parentTableSource = p.getParentTableSource(); } - else if (input instanceof AnalyzeInterceptor ai - && ai.getInput() instanceof se.kuseman.payloadbuilder.core.physicalplan.Projection p) - { - expressions = ProjectionMerger.replace(plan.getExpressions(), p.getExpressions()); - input = p.getInput(); - parentTableSource = p.getParentTableSource(); - } - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.Projection(context.getNextNodeId(), input, schema, expressions, parentTableSource)); + return new se.kuseman.payloadbuilder.core.physicalplan.Projection(context.getNextNodeId(), input, schema, expressions, parentTableSource); } @Override @@ -158,7 +149,7 @@ public IPhysicalPlan visit(Sort plan, Context context) return input; } - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.Sort(context.getNextNodeId(), input, plan.getSortItems())); + return new se.kuseman.payloadbuilder.core.physicalplan.Sort(context.getNextNodeId(), input, plan.getSortItems()); } @Override @@ -197,7 +188,7 @@ public IPhysicalPlan visit(Filter plan, Context context) input = filter.getInput(); } - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.Filter(context.getNextNodeId(), input, new ExpressionPredicate(predicate))); + return new se.kuseman.payloadbuilder.core.physicalplan.Filter(context.getNextNodeId(), input, new ExpressionPredicate(predicate)); } @Override @@ -206,7 +197,7 @@ public IPhysicalPlan visit(Aggregate plan, Context context) IPhysicalPlan input = plan.getInput() .accept(this, context); - return wrapWithAnalyze(context, new HashAggregate(context.getNextNodeId(), input, plan.getAggregateExpressions(), plan.getProjectionExpressions(), plan.getParentTableSource())); + return new HashAggregate(context.getNextNodeId(), input, plan.getAggregateExpressions(), plan.getProjectionExpressions(), plan.getParentTableSource()); } @Override @@ -303,10 +294,9 @@ public IPhysicalPlan visit(TableScan plan, Context context) context.topTableScanVisited = true; - return wrapWithAnalyze(context, - seekPredicate != null - ? new se.kuseman.payloadbuilder.core.physicalplan.IndexSeek(nodeId, plan.getSchema(), plan.getTableSource(), catalog.getName(), seekPredicate, dataSource, plan.getOptions()) - : new se.kuseman.payloadbuilder.core.physicalplan.TableScan(nodeId, plan.getSchema(), plan.getTableSource(), catalog.getName(), dataSource, plan.getOptions())); + return seekPredicate != null + ? new se.kuseman.payloadbuilder.core.physicalplan.IndexSeek(nodeId, plan.getSchema(), plan.getTableSource(), catalog.getName(), seekPredicate, dataSource, plan.getOptions()) + : new se.kuseman.payloadbuilder.core.physicalplan.TableScan(nodeId, plan.getSchema(), plan.getTableSource(), catalog.getName(), dataSource, plan.getOptions()); } @Override @@ -325,14 +315,14 @@ public IPhysicalPlan visit(TableFunctionScan plan, Context context) .getCatalog(catalogAlias); TableFunctionInfo functionInfo = pair.getValue(); - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.TableFunctionScan(context.getNextNodeId(), plan.getSchema(), plan.getTableSource(), catalogAlias, - catalog.getName(), functionInfo, plan.getArguments(), plan.getOptions())); + return new se.kuseman.payloadbuilder.core.physicalplan.TableFunctionScan(context.getNextNodeId(), plan.getSchema(), plan.getTableSource(), catalogAlias, catalog.getName(), functionInfo, + plan.getArguments(), plan.getOptions()); } @Override public IPhysicalPlan visit(ExpressionScan plan, Context context) { - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ExpressionScan(context.getNextNodeId(), plan.getTableSource(), plan.getSchema(), plan.getExpression())); + return new se.kuseman.payloadbuilder.core.physicalplan.ExpressionScan(context.getNextNodeId(), plan.getTableSource(), plan.getSchema(), plan.getExpression()); } @Override @@ -415,8 +405,10 @@ public IPhysicalPlan visit(Join plan, Context context) throw new IllegalArgumentException("RIGHT joins are not supported"); } + // CSOFF IPhysicalPlan outer = plan.getOuter() .accept(this, context); + // CSON IExpression condition = plan.getCondition(); @@ -501,24 +493,28 @@ public IPhysicalPlan visit(Join plan, Context context) } else { - ValueVector forceNoInnerCacheProperty = context.context.getSession() - .getSystemProperty(QuerySession.FORCE_NO_INNER_CACHE); - boolean forceNoInnerCache = !forceNoInnerCacheProperty.isNull(0) - && forceNoInnerCacheProperty.getBoolean(0); - - // We can cache the inner plan if we have a non correlated plain nested loop - if (!pushOuterReference - && !forceNoInnerCache - && !correlated - && !plan.isSwitchedInputs()) - { - inner = wrapWithAnalyze(context, new CachePlan(context.getNextNodeId(), inner)); - } + // Turned off since cache is broken by design atm. + // It's never reset during a statement and this is not correct + // in many cases when there are nested sub query expressions that is cached when they shouldn't + // + // ValueVector forceNoInnerCacheProperty = context.context.getSession() + // .getSystemProperty(QuerySession.FORCE_NO_INNER_CACHE); + // boolean forceNoInnerCache = !forceNoInnerCacheProperty.isNull(0) + // && forceNoInnerCacheProperty.getBoolean(0); + // + // // We can cache the inner plan if we have a non correlated plain nested loop + // if (!pushOuterReference + // && !forceNoInnerCache + // && !correlated + // && !plan.isSwitchedInputs()) + // { + // inner = wrapWithAnalyze(context, new CachePlan(context.getNextNodeId(), inner)); + // } join = createNestedLoop(plan, context.getNextNodeId(), outer, inner, predicate, pushOuterReference); } - return wrapWithAnalyze(context, join); + return join; } @Override @@ -532,7 +528,7 @@ public IPhysicalPlan visit(OperatorFunctionScan plan, Context context) String catalogAlias = pair.getKey(); OperatorFunctionInfo functionInfo = pair.getValue(); - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.OperatorFunctionScan(context.getNextNodeId(), input, functionInfo, catalogAlias, plan.getSchema())); + return new se.kuseman.payloadbuilder.core.physicalplan.OperatorFunctionScan(context.getNextNodeId(), input, functionInfo, catalogAlias, plan.getSchema()); } @Override @@ -540,18 +536,18 @@ public IPhysicalPlan visit(ConstantScan plan, Context context) { if (ConstantScan.ONE_ROW_EMPTY_SCHEMA.equals(plan)) { - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.CONSTANT)); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.CONSTANT); } else if (ConstantScan.ZERO_ROWS_EMPTY_SCHEMA.equals(plan)) { - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.EMPTY)); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.EMPTY); } // Zero row scan with a schema if (plan.getRowsExpressions() .isEmpty()) { - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.of(plan.getSchema()))); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), TupleVector.of(plan.getSchema())); } // Strip all alias expression here since they are not needed anymore @@ -570,10 +566,10 @@ else if (ConstantScan.ZERO_ROWS_EMPTY_SCHEMA.equals(plan)) .allMatch(IExpression::isConstant))) { TupleVector vector = se.kuseman.payloadbuilder.core.physicalplan.ConstantScan.vectorize(plan.getSchema(), rowsExpressions, context.context, false); - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), vector)); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), vector); } - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), plan.getSchema(), rowsExpressions)); + return new se.kuseman.payloadbuilder.core.physicalplan.ConstantScan(context.getNextNodeId(), plan.getSchema(), rowsExpressions); } @Override @@ -581,7 +577,7 @@ public IPhysicalPlan visit(Limit plan, Context context) { IPhysicalPlan input = plan.getInput() .accept(this, context); - return wrapWithAnalyze(context, new se.kuseman.payloadbuilder.core.physicalplan.Limit(context.getNextNodeId(), input, plan.getLimitExpression())); + return new se.kuseman.payloadbuilder.core.physicalplan.Limit(context.getNextNodeId(), input, plan.getLimitExpression()); } @Override @@ -589,7 +585,7 @@ public IPhysicalPlan visit(MaxRowCountAssert plan, Context context) { IPhysicalPlan input = plan.getInput() .accept(this, context); - return wrapWithAnalyze(context, se.kuseman.payloadbuilder.core.physicalplan.Assert.maxRowCount(context.getNextNodeId(), input, plan.getMaxRowCount())); + return se.kuseman.payloadbuilder.core.physicalplan.Assert.maxRowCount(context.getNextNodeId(), input, plan.getMaxRowCount()); } @Override @@ -678,15 +674,6 @@ private TableSource getTopTableSource(ILogicalPlan plan, boolean skipSubQuery) return null; } - static IPhysicalPlan wrapWithAnalyze(Context context, IPhysicalPlan plan) - { - if (context.analyze) - { - return new AnalyzeInterceptor(context.getNextNodeId(), plan); - } - return plan; - } - /** Visitor that collects table source references */ static class TableSourceReferenceCollector extends AExpressionVisitor> { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementPlanner.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementPlanner.java index afef4216..5fb02a55 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementPlanner.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementPlanner.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Map; -import se.kuseman.payloadbuilder.api.QualifiedName; import se.kuseman.payloadbuilder.api.catalog.ISortItem; import se.kuseman.payloadbuilder.api.catalog.TableSchema; import se.kuseman.payloadbuilder.api.execution.IExecutionContext; @@ -34,13 +33,8 @@ static class Context boolean joinPreserveOuterOrder; boolean topTableScanVisited; - Map schemaByTempTable = new HashMap<>(); - Map schemaByTableSource = new HashMap<>(); - /** Flag indicating that next plan to be generated is an analzye */ - boolean analyze; - /** Field with the last created logical plan. Used when holding schema information for temp tables etc. */ ILogicalPlan currentLogicalPlan; @@ -80,16 +74,6 @@ public static QueryStatement plan(QuerySession session, QueryStatement query) ExecutionContext context = new ExecutionContext(session); Context ctx = new Context(context); - // Add existing session tables into context - // This is used when working in a statefull session system like Queryeer - // where the session is reused across executions. - session.getTemporaryTables() - .forEach(e -> ctx.schemaByTempTable.put(e.getKey(), new TableSchema(e.getValue() - .getTupleVector() - .getSchema(), - e.getValue() - .getIndices()))); - return new QueryStatement(query.getStatements() .stream() .map(s -> s.accept(STATEMENT_REWRITER, ctx)) diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementRewriter.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementRewriter.java index d26df128..e04c2815 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementRewriter.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/planning/StatementRewriter.java @@ -9,6 +9,8 @@ import java.util.List; +import org.apache.commons.lang3.ObjectUtils; + import se.kuseman.payloadbuilder.api.QualifiedName; import se.kuseman.payloadbuilder.api.catalog.Catalog; import se.kuseman.payloadbuilder.api.catalog.IDatasink; @@ -36,9 +38,12 @@ import se.kuseman.payloadbuilder.core.logicalplan.Sort; import se.kuseman.payloadbuilder.core.logicalplan.TableScan; import se.kuseman.payloadbuilder.core.logicalplan.optimization.LogicalPlanOptimizer; +import se.kuseman.payloadbuilder.core.parser.Location; import se.kuseman.payloadbuilder.core.parser.ParseException; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeFormat; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnlayzeType; import se.kuseman.payloadbuilder.core.physicalplan.AssignmentPlan; -import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan; import se.kuseman.payloadbuilder.core.physicalplan.IPhysicalPlan; import se.kuseman.payloadbuilder.core.physicalplan.InsertInto; import se.kuseman.payloadbuilder.core.planning.StatementPlanner.Context; @@ -115,11 +120,15 @@ public Statement visit(UseStatement statement, Context context) @Override public Statement visit(DescribeSelectStatement statement, Context context) { - context.analyze = statement.isAnalyze(); PhysicalStatement physicalStatement = (PhysicalStatement) statement.getStatement() .accept(this, context); - context.analyze = false; - return new PhysicalStatement(new DescribePlan(context.getNextNodeId(), physicalStatement.getPlan(), statement.isAnalyze())); + + IPhysicalPlan plan = physicalStatement.getPlan(); + AnalyzeVisitor.AnlayzeType type = statement.isAnalyze() ? AnlayzeType.ANALYZE + : AnlayzeType.DESCRIBE; + String queryText = ObjectUtils.getIfNull(physicalStatement.getLocation(), Location.EMPTY) + .text(); + return new PhysicalStatement(AnalyzeVisitor.describe(plan, type, AnalyzeFormat.TABLE, queryText), null); } @Override @@ -164,7 +173,7 @@ public Statement visit(ShowStatement statement, Context context) new LiteralStringExpression(UTF8String.EMPTY), new LiteralStringExpression(UTF8String.EMPTY)), null), - new Sort(systemFunctionsScan, sortItems)), null), false).accept(this, context); + new Sort(systemFunctionsScan, sortItems)), null), false, Location.EMPTY).accept(this, context); //@formatter:on } @@ -183,8 +192,8 @@ public Statement visit(ShowStatement statement, Context context) QualifiedName qname = isBlank(catalog) ? QualifiedName.of(tableName) : QualifiedName.of(catalog, tableName); TableSourceReference tableSourceRef = new TableSourceReference(2, TableSourceReference.Type.TABLE, "sys", qname, "t"); - return new LogicalSelectStatement(new TableScan(TableSchema.EMPTY, tableSourceRef, se.kuseman.payloadbuilder.api.catalog.DatasourceData.Projection.ALL, emptyList(), null), false).accept(this, - context); + return new LogicalSelectStatement(new TableScan(TableSchema.EMPTY, tableSourceRef, se.kuseman.payloadbuilder.api.catalog.DatasourceData.Projection.ALL, emptyList(), Location.EMPTY), false, + null).accept(this, context); } @Override @@ -264,7 +273,7 @@ private Statement createInsertStatement(LogicalSelectIntoStatement statement, Co : catalog.getInsertIntoSink(context.getSession(), catalogAlias, statement.getTable(), new InsertIntoData(nodeId, schema, options, insertColumns)); //@formatter:on - return new PhysicalStatement(QueryPlanner.wrapWithAnalyze(context, new InsertInto(nodeId, input.getPlan(), insertColumns, sink))); + return new PhysicalStatement(new InsertInto(nodeId, input.getPlan(), insertColumns, sink), statement.getLocation()); } private PhysicalStatement createPhysicalPlan(LogicalSelectStatement statement, Context context) @@ -304,6 +313,6 @@ private PhysicalStatement createPhysicalPlan(LogicalSelectStatement statement, C } // Create a physical plan from the logical - return new PhysicalStatement(physicalPlan); + return new PhysicalStatement(physicalPlan, statement.getLocation()); } } diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/LogicalSelectStatement.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/LogicalSelectStatement.java index 755c7e4f..5bb5e311 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/LogicalSelectStatement.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/LogicalSelectStatement.java @@ -3,17 +3,20 @@ import static java.util.Objects.requireNonNull; import se.kuseman.payloadbuilder.core.logicalplan.ILogicalPlan; +import se.kuseman.payloadbuilder.core.parser.Location; /** Logical select statement. Statement in the planning phase. Will be transformed in to a {@link PhysicalStatement} further down the line */ public class LogicalSelectStatement extends Statement { private final ILogicalPlan select; private final boolean assignmentSelect; + private final Location location; - public LogicalSelectStatement(ILogicalPlan select, boolean assignmentSelect) + public LogicalSelectStatement(ILogicalPlan select, boolean assignmentSelect, Location location) { this.select = requireNonNull(select, "select"); this.assignmentSelect = assignmentSelect; + this.location = location; } public ILogicalPlan getSelect() @@ -26,6 +29,11 @@ public boolean isAssignmentSelect() return assignmentSelect; } + public Location getLocation() + { + return location; + } + @Override public TR accept(StatementVisitor visitor, TC context) { diff --git a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/PhysicalStatement.java b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/PhysicalStatement.java index 76244f68..5ac24c9a 100644 --- a/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/PhysicalStatement.java +++ b/payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/statement/PhysicalStatement.java @@ -2,16 +2,19 @@ import static java.util.Objects.requireNonNull; +import se.kuseman.payloadbuilder.core.parser.Location; import se.kuseman.payloadbuilder.core.physicalplan.IPhysicalPlan; /** Physical statement that wraps a {@link IPhysicalPlan}. */ public class PhysicalStatement extends Statement { private final IPhysicalPlan plan; + private final Location location; - public PhysicalStatement(IPhysicalPlan plan) + public PhysicalStatement(IPhysicalPlan plan, Location location) { this.plan = requireNonNull(plan, "plan"); + this.location = location; } public IPhysicalPlan getPlan() @@ -19,6 +22,11 @@ public IPhysicalPlan getPlan() return plan; } + public Location getLocation() + { + return location; + } + @Override public TR accept(StatementVisitor visitor, TC context) { diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolverTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolverTest.java index 40b17dfc..3d71b191 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolverTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolverTest.java @@ -548,7 +548,7 @@ void test_expand_outer_reference_asterisk_in_sub_query_expression() false, Schema.EMPTY), new OperatorFunctionScan( - Schema.of(ast("__expr0", ResolvedType.table(Schema.of(ast("", "b.*", tableB))), "", null, true)), + Schema.of(col("__expr0", ResolvedType.table(Schema.of(ast("", "b.*", tableB))), true)), projection( ConstantScan.ONE_ROW_EMPTY_SCHEMA, List.of(new AsteriskExpression(QualifiedName.of("b"), null, Set.of(tableB)))), @@ -1020,7 +1020,7 @@ void test_precedence_of_current_outer_matches_asterisk_and_not_asterisk() false, Schema.EMPTY), new OperatorFunctionScan( - Schema.of(ast("__expr0", ResolvedType.table(objectArraySchema), true)), + Schema.of(col("__expr0", ResolvedType.table(objectArraySchema), true)), projection( new ExpressionScan( e_b.withParent(tableB), @@ -1759,7 +1759,7 @@ void test_populate_join_expression_scan_schema_less() false, Schema.EMPTY), new OperatorFunctionScan( - Schema.of(ast("__expr0", ResolvedType.table(subQuerySchema), true)), + Schema.of(col("__expr0", ResolvedType.table(subQuerySchema), true)), projection( new ExpressionScan( e_b.withParent(tableB), @@ -2026,7 +2026,7 @@ void test_that_sub_query_expression_with_table_source_with_unqualified_column_ge new Join( tableScan(schemaSTableA, sTableA), new OperatorFunctionScan( - Schema.of(ast("__expr0", ResolvedType.object(Schema.of(nast("col1", ResolvedType.ANY, tableB), col("col3", Type.Float, sTableA))), true)), + Schema.of(col("__expr0", ResolvedType.object(Schema.of(nast("col1", ResolvedType.ANY, tableB), col("col3", Type.Float, sTableA))), true)), projection( tableScan(schemaB, tableB), List.of( diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/parser/QueryParserTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/parser/QueryParserTest.java index 1d81dd76..597423a4 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/parser/QueryParserTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/parser/QueryParserTest.java @@ -304,7 +304,7 @@ void test_subquery_expression() null), null)), null), - false); + false, null); //@formatter:on Statement actual = s("select a.col, (select a.* from open_table(a) a for objectarray) from \"table\" a where a.col > 10"); @@ -667,7 +667,7 @@ void test_insert_into() void test_select() { // Selects without table source - assertEquals(new LogicalSelectStatement(ConstantScan.create(asList(litInt(1)), null), false), assertSelect("select 1")); + assertEquals(new LogicalSelectStatement(ConstantScan.create(asList(litInt(1)), null), false, null), assertSelect("select 1")); assertSelect("select 1 where false"); assertSelect("select 1 order by 1"); assertSelect("select top 10 1"); @@ -682,7 +682,7 @@ void test_select() null), List.of(new AsteriskExpression(null)), null), - false), assertSelect("select * from (a.b) a")); + false, null), assertSelect("select * from (a.b) a")); //@formatter:on assertSelectFail(ParseException.class, "Expression scans cannot have options", "select * from (a.b) a with (a=123)"); diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanTest.java index 842a76cb..c7be39d9 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/APhysicalPlanTest.java @@ -149,6 +149,12 @@ public List getChildren() return emptyList(); } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + throw new RuntimeException("Cannot be visited"); + } + @Override public TupleIterator execute(IExecutionContext context) { diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/ProjectionTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/ProjectionTest.java index c4b36bae..7f281d82 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/ProjectionTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/physicalplan/ProjectionTest.java @@ -151,6 +151,12 @@ public int getNodeId() return 0; } + @Override + public T accept(IPhysicalPlanVisitor visitor, C context) + { + throw new RuntimeException("Cannot be visited"); + } + @Override public List getChildren() { diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerAnalyzeTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerAnalyzeTest.java new file mode 100644 index 00000000..f2a54071 --- /dev/null +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerAnalyzeTest.java @@ -0,0 +1,358 @@ +package se.kuseman.payloadbuilder.core.planning; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static se.kuseman.payloadbuilder.api.utils.MapUtils.entry; +import static se.kuseman.payloadbuilder.api.utils.MapUtils.ofEntries; +import static se.kuseman.payloadbuilder.core.planning.QueryPlannerTest.parse; + +import java.util.List; + +import org.apache.commons.lang3.tuple.Triple; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import se.kuseman.payloadbuilder.api.QualifiedName; +import se.kuseman.payloadbuilder.api.catalog.Column.Type; +import se.kuseman.payloadbuilder.api.catalog.FunctionInfo; +import se.kuseman.payloadbuilder.api.catalog.IDatasource; +import se.kuseman.payloadbuilder.api.catalog.IPredicate; +import se.kuseman.payloadbuilder.api.catalog.ResolvedType; +import se.kuseman.payloadbuilder.api.catalog.ScalarFunctionInfo; +import se.kuseman.payloadbuilder.api.catalog.Schema; +import se.kuseman.payloadbuilder.core.catalog.CoreColumn; +import se.kuseman.payloadbuilder.core.catalog.TableSourceReference; +import se.kuseman.payloadbuilder.core.expression.FunctionCallExpression; +import se.kuseman.payloadbuilder.core.expression.InExpression; +import se.kuseman.payloadbuilder.core.expression.LikeExpression; +import se.kuseman.payloadbuilder.core.expression.LiteralStringExpression; +import se.kuseman.payloadbuilder.core.expression.NullPredicateExpression; +import se.kuseman.payloadbuilder.core.physicalplan.APhysicalPlanTest; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeInterceptor; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeFormat; +import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnlayzeType; +import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan; +import se.kuseman.payloadbuilder.core.physicalplan.ExpressionPredicate; +import se.kuseman.payloadbuilder.core.physicalplan.Filter; +import se.kuseman.payloadbuilder.core.physicalplan.IPhysicalPlan; +import se.kuseman.payloadbuilder.core.physicalplan.Projection; +import se.kuseman.payloadbuilder.core.physicalplan.TableScan; +import se.kuseman.payloadbuilder.core.planning.QueryPlannerTest.TestCatalog; +import se.kuseman.payloadbuilder.core.statement.PhysicalStatement; +import se.kuseman.payloadbuilder.core.statement.QueryStatement; +import se.kuseman.payloadbuilder.core.utils.CollectionUtils; + +/** Query planner test with analyze option. */ +class QueryPlannerAnalyzeTest extends APhysicalPlanTest +{ + @BeforeEach + void setup() + { + session.setDefaultCatalogAlias("t"); + } + + @Test + void test_analyze_dont_describe_describe_plans() + { + //@formatter:off + IPhysicalPlan plan = new DescribePlan( + 11, + new AnalyzeInterceptor( + 10, + new TableScan( + 0, + Schema.of(ast("", table)), + table, + "test", + mock(IDatasource.class), + emptyList())), + true, + AnalyzeFormat.JSON, + ""); + //@formatter:on + assertSame(plan, AnalyzeVisitor.describe(plan, AnlayzeType.ANALYZE, AnalyzeFormat.JSON, "")); + } + + @Test + void test_analyze_dont_intercept_analyze_interceptors() + { + //@formatter:off + IPhysicalPlan plan = new DescribePlan( + 11, + new AnalyzeInterceptor( + 10, + new TableScan( + 0, + Schema.of(ast("", table)), + table, + "test", + mock(IDatasource.class), + emptyList())), + true, + AnalyzeFormat.JSON, + ""); + //@formatter:on + assertSame(plan.getChildren() + .get(0), + AnalyzeVisitor.describe(plan.getChildren() + .get(0), AnlayzeType.ANALYZE, AnalyzeFormat.JSON, "") + .getChildren() + .get(0)); + } + + @Test + void test_describe_tableScan() + { + //@formatter:off + String query = """ + describe + select * + from tableA + """; + //@formatter:on + + TestCatalog t = new TestCatalog(emptyMap()); + catalogRegistry.registerCatalog("t", t); + + QueryStatement queryStatement = parse(query); + queryStatement = StatementPlanner.plan(session, queryStatement); + + IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() + .get(0)).getPlan(); + + TableSourceReference table = new TableSourceReference(0, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), ""); + + //@formatter:off + IPhysicalPlan expected = + new DescribePlan( + 1, + new TableScan( + 0, + Schema.of(ast("", table)), + table, + "test", + t.scanDataSources.get(0), + emptyList()), + false, AnalyzeFormat.TABLE, ""); + //@formatter:on + + // System.out.println(actual.print(0)); + // System.out.println(expected.print(0)); + + Assertions.assertThat(actual) + .usingRecursiveComparison() + .ignoringFields("queryText") + .isEqualTo(expected); + + assertEquals(expected, actual); + } + + @Test + void test_sub_query_filter_elimination_analyze() + { + // Verify that the nested filters are merged when the subquery are eliminated + //@formatter:off + String query = """ + analyze + select * + from + ( + select * + from tableA a + where a.col1 > 10 + ) x + where x.col > 20 + """; + //@formatter:on + + TestCatalog t = new TestCatalog(emptyMap()); + catalogRegistry.registerCatalog("t", t); + + QueryStatement queryStatement = parse(query); + queryStatement = StatementPlanner.plan(session, queryStatement); + + IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() + .get(0)).getPlan(); + + TableSourceReference tableA = new TableSourceReference(1, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), "a"); + + Schema expectedSchemaA = Schema.of(ast("a", tableA)); + + //@formatter:off + IPhysicalPlan expected = + new DescribePlan(4, + new AnalyzeInterceptor( + 3, + new Filter( + 1, + new AnalyzeInterceptor( + 2, + new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()) + ), + new ExpressionPredicate(and(gt(cre("col1", tableA, CoreColumn.Type.NAMED_ASTERISK), intLit(10)), + gt(cre("col", tableA, CoreColumn.Type.NAMED_ASTERISK), intLit(20)))) + )), + true, AnalyzeFormat.TABLE, ""); + //@formatter:on + + // System.out.println(actual.print(0)); + // System.out.println(expected.print(0)); + + Assertions.assertThat(actual) + .usingRecursiveComparison() + .ignoringFields("queryText") + .isEqualTo(expected); + + assertEquals(expected, actual); + } + + @Test + void test_sub_query_projection_elimination_analyze() + { + // Verify that the nested projections are merged when the subquery are eliminated + //@formatter:off + String query = """ + analyze + select x.col1, x.col2, x.col3 + x.col2 + from + ( + select col1, col2, col3 + from tableA + ) x + """; + //@formatter:on + + TestCatalog t = new TestCatalog(emptyMap()); + catalogRegistry.registerCatalog("t", t); + + QueryStatement queryStatement = parse(query); + queryStatement = StatementPlanner.plan(session, queryStatement); + + IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() + .get(0)).getPlan(); + + TableSourceReference tableA = new TableSourceReference(1, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), ""); + TableSourceReference subQueryX = new TableSourceReference(0, TableSourceReference.Type.SUBQUERY, "", QualifiedName.of("x"), "x"); + Schema expectedSchemaA = Schema.of(ast("", tableA)); + + //@formatter:off + IPhysicalPlan expected = + new DescribePlan( + 5, + new AnalyzeInterceptor( + 4, + new Projection( + 2, + new AnalyzeInterceptor( + 3, + new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()) + ), + Schema.of( + nast("col1", ResolvedType.ANY, tableA), + nast("col2", ResolvedType.ANY, tableA), + col("", Type.Any, "col3 + col2") + ), + List.of(cre("col1", tableA, CoreColumn.Type.NAMED_ASTERISK), cre("col2", tableA, CoreColumn.Type.NAMED_ASTERISK), + add(cre("col3", tableA, CoreColumn.Type.NAMED_ASTERISK), cre("col2", tableA, CoreColumn.Type.NAMED_ASTERISK))), + subQueryX + ) + ), + true, AnalyzeFormat.TABLE, ""); + //@formatter:on + + // System.out.println(actual.print(0)); + // System.out.println(expected.print(0)); + + Assertions.assertThat(actual) + .usingRecursiveComparison() + .ignoringFields("queryText") + .isEqualTo(expected); + + assertEquals(expected, actual); + } + + @Test + void test_predicate_pushdown_with_analyze() + { + //@formatter:off + String query = "" + + "analyze select * " + + "from \"table\" " + + "where col = 10 " // Comparison + + "and col2 in (10,20) " // In + + "and col3 like 'string' " // Like + + "and col4 is null " // Null + + "and someFunc() "; // Function call + + //@formatter:on + + // Capture col and "" (someFunc) + ScalarFunctionInfo someFunc = new ScalarFunctionInfo("someFunc", FunctionInfo.FunctionType.SCALAR) + { + }; + TestCatalog catalog = new TestCatalog(ofEntries(entry(QualifiedName.of("table"), CollectionUtils.asSet("col", "")))) + { + { + registerFunction(someFunc); + } + }; + + catalogRegistry.registerCatalog("t", catalog); + + QueryStatement queryStatement = parse(query); + queryStatement = StatementPlanner.plan(session, queryStatement); + + IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() + .get(0)).getPlan(); + + TableSourceReference table = new TableSourceReference(0, TableSourceReference.Type.TABLE, "", QualifiedName.of("table"), ""); + + //@formatter:off + Schema expectedSchema = Schema.of(ast("", table)); + + IPhysicalPlan expected = new DescribePlan( + 4, + new AnalyzeInterceptor( + 3, + new Filter( + 1, + new AnalyzeInterceptor( + 2, + new TableScan(0, expectedSchema, table, "test", catalog.scanDataSources.get(0), emptyList()) + ), + new ExpressionPredicate( + and( + and( + new InExpression(cre("col2", table, CoreColumn.Type.NAMED_ASTERISK), asList(intLit(10), intLit(20)), false), + new LikeExpression(cre("col3", table, CoreColumn.Type.NAMED_ASTERISK), new LiteralStringExpression("string"), false, null) + ), + new NullPredicateExpression(cre("col4", table, CoreColumn.Type.NAMED_ASTERISK), false))) + )), + true, AnalyzeFormat.TABLE, ""); + //@formatter:on + + //@formatter:off + assertEquals(1, catalog.consumedPredicate.size()); + assertEquals(asList( + Triple.of(null, IPredicate.Type.FUNCTION_CALL, asList(new FunctionCallExpression("t", someFunc, null, asList()))), + Triple.of(QualifiedName.of("col"), IPredicate.Type.COMPARISION, asList(intLit(10))) + ), catalog.consumedPredicate.get(QualifiedName.of("table"))); + //@formatter:on + + // System.out.println(actual.print(0)); + // System.out.println(expected.print(0)); + + Assertions.assertThat(actual) + .usingRecursiveComparison() + .ignoringFields("queryText") + .isEqualTo(expected); + + assertEquals(expected, actual); + } +} diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java index c195097c..3f6f54c9 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/planning/QueryPlannerTest.java @@ -70,11 +70,8 @@ import se.kuseman.payloadbuilder.core.expression.VariableExpression; import se.kuseman.payloadbuilder.core.parser.Location; import se.kuseman.payloadbuilder.core.physicalplan.APhysicalPlanTest; -import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeInterceptor; import se.kuseman.payloadbuilder.core.physicalplan.Assert; -import se.kuseman.payloadbuilder.core.physicalplan.CachePlan; import se.kuseman.payloadbuilder.core.physicalplan.ConstantScan; -import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan; import se.kuseman.payloadbuilder.core.physicalplan.ExpressionPredicate; import se.kuseman.payloadbuilder.core.physicalplan.ExpressionScan; import se.kuseman.payloadbuilder.core.physicalplan.Filter; @@ -171,7 +168,7 @@ void test_cache_inner_plan_regression() ocre("campaign", variant, CoreColumn.Type.NAMED_ASTERISK)), SystemCatalog.get().getOperatorFunction("object_array"), "sys", - Schema.of(ast("__expr1", ResolvedType.table(campsSchema), true))), + Schema.of(col("__expr1", ResolvedType.table(campsSchema), true))), Set.of(nast("campaign", ResolvedType.ANY, variant)), null, Schema.of(ast("v", ResolvedType.ANY, variant))), @@ -180,7 +177,7 @@ void test_cache_inner_plan_regression() null), SystemCatalog.get().getOperatorFunction("object"), "sys", - Schema.of(ast("__expr0", ResolvedType.object(priceSchema), true))), + Schema.of(col("__expr0", ResolvedType.object(priceSchema), true))), Set.of(nast("campaign", ResolvedType.ANY, variant)), null, Schema.of(ast("v", variant))), @@ -240,9 +237,9 @@ void test_cache_inner_plan_with_nested_complex_structure() col("camps", ResolvedType.table(constantSchema)) ); IPhysicalPlan expected = new Projection( - 10, + 8, NestedLoop.leftJoin( - 9, + 7, new TableScan( 0, variantSchema, @@ -250,33 +247,27 @@ void test_cache_inner_plan_with_nested_complex_structure() "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 8, - new OperatorFunctionScan( - 7, - new Projection( - 6, - NestedLoop.leftJoin( - 5, - new ConstantScan(1, TupleVector.CONSTANT), - new CachePlan( - 4, - new OperatorFunctionScan( - 3, - new ConstantScan(2, TupleVector.of(constantSchema)), - SystemCatalog.get().getOperatorFunction("object_array"), - "sys", - Schema.of(col("__expr1", ResolvedType.table(constantSchema), true))) - ), - null, - false), - Schema.of(col("camps", ResolvedType.table(constantSchema))), - List.of(new AliasExpression(ce("__expr1", 0, ResolvedType.table(constantSchema)), "camps")), - null), - SystemCatalog.get().getOperatorFunction("object"), - "sys", - Schema.of(col("__expr0", ResolvedType.object(priceSchema), true))) - ), + new OperatorFunctionScan( + 6, + new Projection( + 5, + NestedLoop.leftJoin( + 4, + new ConstantScan(1, TupleVector.CONSTANT), + new OperatorFunctionScan( + 3, + new ConstantScan(2, TupleVector.of(constantSchema)), + SystemCatalog.get().getOperatorFunction("object_array"), + "sys", + Schema.of(col("__expr1", ResolvedType.table(constantSchema), true))), + null, + false), + Schema.of(col("camps", ResolvedType.table(constantSchema))), + List.of(new AliasExpression(ce("__expr1", 0, ResolvedType.table(constantSchema)), "camps")), + null), + SystemCatalog.get().getOperatorFunction("object"), + "sys", + Schema.of(col("__expr0", ResolvedType.object(priceSchema), true))), null, false), Schema.of(col("price", ResolvedType.object(priceSchema))), @@ -342,63 +333,6 @@ void test_sub_query_filter_elimination() assertEquals(expected, actual); } - @Test - void test_sub_query_filter_elimination_analyze() - { - // Verify that the nested filters are merged when the subquery are eliminated - //@formatter:off - String query = """ - analyze - select * - from - ( - select * - from tableA a - where a.col1 > 10 - ) x - where x.col > 20 - """; - //@formatter:on - - TestCatalog t = new TestCatalog(emptyMap()); - catalogRegistry.registerCatalog("t", t); - - QueryStatement queryStatement = parse(query); - queryStatement = StatementPlanner.plan(session, queryStatement); - - IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() - .get(0)).getPlan(); - - TableSourceReference tableA = new TableSourceReference(1, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), "a"); - - Schema expectedSchemaA = Schema.of(ast("a", tableA)); - - //@formatter:off - IPhysicalPlan expected = new DescribePlan(4, - new AnalyzeInterceptor( - 3, - new Filter( - 2, - new AnalyzeInterceptor( - 1, - new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()) - ), - new ExpressionPredicate(and(gt(cre("col1", tableA, CoreColumn.Type.NAMED_ASTERISK), intLit(10)), - gt(cre("col", tableA, CoreColumn.Type.NAMED_ASTERISK), intLit(20)))) - )), - true); - //@formatter:on - - // System.out.println(actual.print(0)); - // System.out.println(expected.print(0)); - - Assertions.assertThat(actual) - .usingRecursiveComparison() - .isEqualTo(expected); - - assertEquals(expected, actual); - } - @Test void test_nested_sub_query_projection_elimination() { @@ -512,70 +446,6 @@ void test_sub_query_projection_elimination() assertEquals(expected, actual); } - @Test - void test_sub_query_projection_elimination_analyze() - { - // Verify that the nested projections are merged when the subquery are eliminated - //@formatter:off - String query = """ - analyze - select x.col1, x.col2, x.col3 + x.col2 - from - ( - select col1, col2, col3 - from tableA - ) x - """; - //@formatter:on - - TestCatalog t = new TestCatalog(emptyMap()); - catalogRegistry.registerCatalog("t", t); - - QueryStatement queryStatement = parse(query); - queryStatement = StatementPlanner.plan(session, queryStatement); - - IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() - .get(0)).getPlan(); - - TableSourceReference tableA = new TableSourceReference(1, TableSourceReference.Type.TABLE, "", QualifiedName.of("tableA"), ""); - TableSourceReference subQueryX = new TableSourceReference(0, TableSourceReference.Type.SUBQUERY, "", QualifiedName.of("x"), "x"); - Schema expectedSchemaA = Schema.of(ast("", tableA)); - - //@formatter:off - IPhysicalPlan expected = - new DescribePlan( - 6, - new AnalyzeInterceptor( - 5, - new Projection( - 4, - new AnalyzeInterceptor( - 1, - new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()) - ), - Schema.of( - nast("col1", ResolvedType.ANY, tableA), - nast("col2", ResolvedType.ANY, tableA), - col("", Type.Any, "col3 + col2") - ), - List.of(cre("col1", tableA, CoreColumn.Type.NAMED_ASTERISK), cre("col2", tableA, CoreColumn.Type.NAMED_ASTERISK), - add(cre("col3", tableA, CoreColumn.Type.NAMED_ASTERISK), cre("col2", tableA, CoreColumn.Type.NAMED_ASTERISK))), - subQueryX - ) - ), - true); - //@formatter:on - - // System.out.println(actual.print(0)); - // System.out.println(expected.print(0)); - - Assertions.assertThat(actual) - .usingRecursiveComparison() - .isEqualTo(expected); - - assertEquals(expected, actual); - } - @Test void test_sub_query_projection_elimination_2() { @@ -866,12 +736,9 @@ void test_using_same_table_ref_different_places_non_equi() //@formatter:off IPhysicalPlan expected = NestedLoop.innerJoin( - 3, + 2, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 2, - new TableScan(1, expectedSchemaA1, tableA1, "test", t.scanDataSources.get(1), emptyList()) - ), + new TableScan(1, expectedSchemaA1, tableA1, "test", t.scanDataSources.get(1), emptyList()), new ExpressionPredicate(gt(cre("value", tableA1, ResolvedType.of(Type.Any), CoreColumn.Type.NAMED_ASTERISK), cre("value", tableA, ResolvedType.of(Type.Any), CoreColumn.Type.NAMED_ASTERISK))), null, @@ -917,12 +784,9 @@ void test_outer_apply_non_correlated() Schema expectedSchemaB = Schema.of(ast("b", tableB)); //@formatter:off IPhysicalPlan expected = NestedLoop.leftJoin( - 3, + 2, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()) - ), + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), null, false); //@formatter:on @@ -1079,15 +943,12 @@ public TableSchema getTableSchema(IExecutionContext context, String catalogAlias //@formatter:off IPhysicalPlan expected = NestedLoop.innerJoin( - 4, + 3, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 3, - new Filter( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), - new ExpressionPredicate(eq(cre("active", tableB, CoreColumn.Type.NAMED_ASTERISK), LiteralBooleanExpression.TRUE))) - ), + new Filter( + 2, + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), + new ExpressionPredicate(eq(cre("active", tableB, CoreColumn.Type.NAMED_ASTERISK), LiteralBooleanExpression.TRUE))), null, false); //@formatter:on @@ -1128,7 +989,7 @@ public TableSchema getTableSchema(IExecutionContext context, String catalogAlias }; catalogRegistry.registerCatalog("t", t); - session.setSystemProperty(QuerySession.FORCE_NO_INNER_CACHE, ValueVector.literalBoolean(true, 1)); + // session.setSystemProperty(QuerySession.FORCE_NO_INNER_CACHE, ValueVector.literalBoolean(true, 1)); QueryStatement queryStatement = parse(query); queryStatement = StatementPlanner.plan(session, queryStatement); @@ -1383,14 +1244,11 @@ public IDatasource getScanDataSource(IQuerySession session, String catalogAlias, //@formatter:off IPhysicalPlan expected = new Sort( - 4, + 3, NestedLoop.innerJoin( - 3, + 2, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()) - ), + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), null, false), asList(sortItem(cre("col", tableA, CoreColumn.Type.NAMED_ASTERISK), Order.DESC, NullOrder.UNDEFINED), @@ -1451,14 +1309,11 @@ public IDatasource getScanDataSource(IQuerySession session, String catalogAlias, //@formatter:off IPhysicalPlan expected = new Sort( - 4, + 3, NestedLoop.innerJoin( - 3, + 2, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()) - ), + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), null, false), asList(sortItem(cre("col2", tableB, CoreColumn.Type.NAMED_ASTERISK), Order.ASC, NullOrder.UNDEFINED))); @@ -1592,24 +1447,21 @@ void test_sub_query_expression() //@formatter:off IPhysicalPlan expected = new Projection( - 6, + 5, NestedLoop.leftJoin( - 5, + 4, new TableScan(0, expectedSchemaA, tableA, "test", t.scanDataSources.get(0), emptyList()), - new CachePlan( - 4, - Assert.maxRowCount( - 3, - new Projection( - 2, - new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), - Schema.of( - nast("__expr0", ResolvedType.ANY, "", tableB, true, "col1") - ), - asList(new AliasExpression(cre("col1", tableB, CoreColumn.Type.NAMED_ASTERISK), "__expr0", true)), - null), - 1) - ), + Assert.maxRowCount( + 3, + new Projection( + 2, + new TableScan(1, expectedSchemaB, tableB, "test", t.scanDataSources.get(1), emptyList()), + Schema.of( + nast("__expr0", ResolvedType.ANY, "", tableB, true, "col1") + ), + asList(new AliasExpression(cre("col1", tableB, CoreColumn.Type.NAMED_ASTERISK), "__expr0", true)), + null), + 1), null, false), Schema.of( @@ -1702,7 +1554,7 @@ void test_sub_query_expression_with_switched_inputs_with_for() null), SystemCatalog.get().getOperatorFunction("object_array"), "sys", - Schema.of(ast("__expr0", ResolvedType.table(objectArraySchema), true))), + Schema.of(col("__expr0", ResolvedType.table(objectArraySchema), true))), asSet(pop("b", ResolvedType.table(expectedSchemaB), tableB)), null, SchemaUtils.joinSchema(expectedSchemaA, expectedSchemaB, "b")), @@ -3023,84 +2875,6 @@ void test_predicate_pushdown_2() assertEquals(expected, actual); } - @Test - void test_predicate_pushdown_with_analyze() - { - //@formatter:off - String query = "" - + "analyze select * " - + "from \"table\" " - + "where col = 10 " // Comparison - + "and col2 in (10,20) " // In - + "and col3 like 'string' " // Like - + "and col4 is null " // Null - + "and someFunc() "; // Function call - - //@formatter:on - - // Capture col and "" (someFunc) - ScalarFunctionInfo someFunc = new ScalarFunctionInfo("someFunc", FunctionInfo.FunctionType.SCALAR) - { - }; - TestCatalog catalog = new TestCatalog(ofEntries(entry(QualifiedName.of("table"), CollectionUtils.asSet("col", "")))) - { - { - registerFunction(someFunc); - } - }; - - catalogRegistry.registerCatalog("t", catalog); - - QueryStatement queryStatement = parse(query); - queryStatement = StatementPlanner.plan(session, queryStatement); - - IPhysicalPlan actual = ((PhysicalStatement) queryStatement.getStatements() - .get(0)).getPlan(); - - TableSourceReference table = new TableSourceReference(0, TableSourceReference.Type.TABLE, "", QualifiedName.of("table"), ""); - - //@formatter:off - Schema expectedSchema = Schema.of(ast("", table)); - - IPhysicalPlan expected = new DescribePlan( - 4, - new AnalyzeInterceptor( - 3, - new Filter( - 2, - new AnalyzeInterceptor( - 1, - new TableScan(0, expectedSchema, table, "test", catalog.scanDataSources.get(0), emptyList()) - ), - new ExpressionPredicate( - and( - and( - new InExpression(cre("col2", table, CoreColumn.Type.NAMED_ASTERISK), asList(intLit(10), intLit(20)), false), - new LikeExpression(cre("col3", table, CoreColumn.Type.NAMED_ASTERISK), new LiteralStringExpression("string"), false, null) - ), - new NullPredicateExpression(cre("col4", table, CoreColumn.Type.NAMED_ASTERISK), false))) - )), - true); - //@formatter:on - - //@formatter:off - assertEquals(1, catalog.consumedPredicate.size()); - assertEquals(asList( - Triple.of(null, IPredicate.Type.FUNCTION_CALL, asList(new FunctionCallExpression("t", someFunc, null, asList()))), - Triple.of(QualifiedName.of("col"), IPredicate.Type.COMPARISION, asList(intLit(10))) - ), catalog.consumedPredicate.get(QualifiedName.of("table"))); - //@formatter:on - - // System.out.println(actual.print(0)); - // System.out.println(expected.print(0)); - - Assertions.assertThat(actual) - .usingRecursiveComparison() - .isEqualTo(expected); - - assertEquals(expected, actual); - } - @Test void test_predicate_pushdown_all_filters_consumed() { @@ -3151,7 +2925,7 @@ void test_predicate_pushdown_all_filters_consumed() assertEquals(expected, actual); } - private QueryStatement parse(String query) + static QueryStatement parse(String query) { return PARSER.parseQuery(query, null); } diff --git a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/test/TestHarnessRunner.java b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/test/TestHarnessRunner.java index 341a1f02..7f9efd43 100644 --- a/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/test/TestHarnessRunner.java +++ b/payloadbuilder-core/src/test/java/se/kuseman/payloadbuilder/core/test/TestHarnessRunner.java @@ -389,8 +389,31 @@ private void assertResultSetEqual(boolean onlyAssertExpectedColumns, boolean sch { ColumnValue expectedColumn = j < expectedRow.size() ? expectedRow.get(j) : null; - ColumnValue actualColumn = j < actualRow.size() ? actualRow.get(j) - : null; + + ColumnValue actualColumn = null; + // Find the expected column among the actual + if (onlyAssertExpectedColumns) + { + for (int k = j; k < actualRow.size(); k++) + { + if (expectedColumn.getKey() + .equalsIgnoreCase(actualRow.get(k) + .getKey())) + { + actualColumn = actualRow.get(k); + } + } + if (actualColumn == null) + { + throw new IllegalArgumentException("No column found among actual with name: " + expectedColumn.getKey()); + } + } + else + { + actualColumn = j < actualRow.size() ? actualRow.get(j) + : null; + + } if (!equals(expectedColumn, actualColumn)) { diff --git a/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json b/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json index ca784787..444c066b 100644 --- a/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json +++ b/payloadbuilder-core/src/test/resources/harnessCases/BaseConstructs.json @@ -112,14 +112,14 @@ "name": "table", "columns": [ "id", "table", "object", "table2" ], "types": [ - {"type": "String"}, + { "type": "String"}, { "type": "String" }, { "type": "Object", "schema": [ { "name": "key", "type": { "type": "Int" } } ] }, { "type": "Table", "schema": [ { "name": "key", "type": { "type": "Int" } }, { "name": "key2", "type": { "type": "Boolean" } } ] } ], "rows": [ - [ "1", "[{ \"key\": 123, \"value\": 456 },{ \"key\": 789, \"value\": 101112 }]", { "key": 666 }, [{ "key": 4567, "key2": true },{ "key": 891011, "key2": false }] ], - [ "2", "[]", null, [{ "key": 666, "key2": false },{ "key": 1337, "key2": true }] ] + [ "1", "[{ \"key\": 123, \"value\": 456 },{ \"key\": 789, \"value\": 101112 }]", { "key": 666 }, [ { "key": 4567, "key2": true } , { "key": 891011, "key2": false } ] ], + [ "2", "[]", null, [ { "key": 666, "key2": false } , { "key": 1337, "key2": true } ] ] ] } ] @@ -809,6 +809,7 @@ }, { "name": "Describe select, schema full, any vectors", + "onlyAssertExpectedColumns": true, "schemaLess": false, "typedVectors": false, "query": [ @@ -822,18 +823,19 @@ ], "expectedResultSets": [ [ - [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], - [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Hash Time", "value": "00:00:00.000" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Hash Time", "value": "00:00:00.000" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] ] ] }, { - "name": "Analyze select, schema full", - "ignore": true, - "_todo": "Add support for regex assert columns, this query contains times that isn't deterministic", - "schemaLess": false, + "name": "Analyze select, schema less, any vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": true, + "typedVectors": false, "query": [ "analyze ", "select d.id ", @@ -845,15 +847,129 @@ ], "expectedResultSets": [ [ - [{ "key": "Node Id", "value": 3 }, { "key": "Name", "value": "+- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 2 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], - [{ "key": "Node Id", "value": 1 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Hash Time", "value": "00:00:00.000" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 0 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + [{ "key": "Node Id", "value": 8 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": 8 }, { "key": "Name", "value": "| +- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "d (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + ] + ] + }, + { + "name": "Analyze select 2, schema full, any vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": false, + "query": [ + "analyze ", + "select * ", + "from data d", + "inner join source s ", + " on s.col1 = d.id " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 6 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 6 }, { "key": "Name", "value": "| +- Hash Match" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any), col1 (Any), col2 (Any)" }, { "key": "Catalog", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Any), row_id (Any), data (Any)" }, { "key": "Catalog", "value": "Test#c" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: source" }, { "key": "Output", "value": "col1 (Any), col2 (Any)" }, { "key": "Catalog", "value": "Test#c" }] + ] + ] + }, + { + "name": "Analyze select 3, schema full, types vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": true, + "query": [ + "set @@force_nested_loop=true ", + "analyze ", + "select * ", + "from data d", + "inner join source s ", + " on s.col1 = d.id " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 6 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 6 }, { "key": "Name", "value": "| +- Nested Loop" }, { "key": "Output", "value": "id (Int), row_id (Int), data (String), col1 (Int), col2 (Int)" }, { "key": "Catalog", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "id (Int), row_id (Int), data (String)" }, { "key": "Catalog", "value": "Test#c" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: source" }, { "key": "Output", "value": "col1 (Int), col2 (Int)" }, { "key": "Catalog", "value": "Test#c" }] + ] + ] + }, + { + "name": "Analyze select 4, schema full, types vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": true, + "query": [ + "analyze ", + "select top 1 * ", + "from (", + " values (1,2), (3,4) ", + ") x (col1, col2) " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": null } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Limit" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Constant Scan" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 2}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ] + ] + ] + }, + { + "name": "Analyze select 5, dynamic no extended output, schema full, types vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": true, + "query": [ + "set @@plan_analyze = true ", + "set @@plan_format = 'table' ", + "set @@plan_extended_output = false ", + + "", + "select top 1 * ", + "from (", + " values (1,2), (3,4) ", + ") x (col1, col2) " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": null } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Limit" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Constant Scan" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 2}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ] + ] + ] + }, + { + "name": "Analyze select 6, dynamic extended output, schema full, types vectors", + "onlyAssertExpectedColumns": true, + "schemaLess": false, + "typedVectors": true, + "query": [ + "set @@plan_analyze = true ", + "set @@plan_format = 'table' ", + "", + "select top 1 * ", + "from (", + " values (1,2), (3,4) ", + ") x (col1, col2) " + ], + "expectedResultSets": [ + [ + [{ "key": "col1", "value": 1 }, { "key": "col2", "value": 2 } ] + ], + [ + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": null } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Limit" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 1}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Constant Scan" }, { "key": "Execution Count", "value": 1}, { "key": "Processed Rows", "value": 2}, { "key": "Output", "value": "col1 (Int), col2 (Int)" } ] ] ] }, { "name": "Describe select, schema less", + "onlyAssertExpectedColumns": true, "schemaLess": true, "query": [ "describe ", @@ -866,10 +982,36 @@ ], "expectedResultSets": [ [ - [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], - [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Hash Time", "value": "00:00:00.000" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], - [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "d (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Hash Time", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "d (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] + ] + ] + }, + { + "name": "Describe select, dynamic no extended output, schema less", + "onlyAssertExpectedColumns": true, + "schemaLess": true, + "query": [ + "set @@plan_describe = true ", + "set @@plan_extended_output = false ", + "", + "select d.id ", + ", max(row_id) ", + "from data d", + "group by d.id ", + "having count(1) = 1 ", + "order by id " + ], + "expectedResultSets": [ + [ + [{ "key": "Node Id", "value": 4 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" }, { "key": "Output", "value": null }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": 4 }, { "key": "Name", "value": "| +- Sort" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": "[d.id ASC]" }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Filter" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": "__expr0 = 1" }], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Hash Aggregate" }, { "key": "Output", "value": "id (Any), max(d.row_id) (Any)" }, { "key": "Catalog", "value": null }, { "key": "Defined Values", "value": "__expr0: count(1)" }, { "key": "Group By", "value": "[d.id]" }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 1 }, { "key": "Name", "value": "| +- Scan: data" }, { "key": "Output", "value": "d (Any)" }, { "key": "Catalog", "value": "Test#c" }, { "key": "Defined Values", "value": null }, { "key": "Group By", "value": null }, { "key": "Order By", "value": null }, { "key": "Predicate", "value": null }] ] ] }, @@ -2620,6 +2762,44 @@ [ { "key": "_outer", "value": { "_inner": [ { "key": 666, "key2": false }, { "key": 1337, "key2": true } ] } } ] ] ] + }, + { + "name": "Sub query expressions with no rows", + "description": [ + "Regression test of a query with sub query expressions that returned no rows along with ordinal resolved columns ", + "that got mismatch schemas compile time vs runtime" + ], + "query": [ + "--set @@printplan=true ", + "use complexColumns ", + "SELECT ", + "( ", + " SELECT ", + " ( ", + " SELECT * ", + " FROM (totable(v.table2)) x ", + " WHERE x.key = 666 ", + " FOR OBJECT_ARRAY ", + " ) _outer ", + " ,( ", + " SELECT key * 2 newKey ", + " FROM (totable(v.table2)) x ", + " FOR OBJECT_ARRAY ", + " ) _outer2 ", + " , v.id ", + " FOR OBJECT ", + ") x ", + "FROM ( ", + " select x.table2, x.id ", + " from \"table\" x ", + ") v " + ], + "expectedResultSets": [ + [ + [ { "key": "x", "value": { "_outer": null, "_outer2": [ { "newKey": 9134}, { "newKey": 1782022 } ], "id": "1" } } ], + [ { "key": "x", "value": { "_outer": [ { "key": 666, "key2": false } ], "_outer2": [ { "newKey": 1332}, { "newKey": 2674 } ], "id": "2" } } ] + ] + ] } ] } diff --git a/payloadbuilder-core/src/test/resources/harnessCases/TemporaryTables.json b/payloadbuilder-core/src/test/resources/harnessCases/TemporaryTables.json index 446e27c8..983cc4d0 100644 --- a/payloadbuilder-core/src/test/resources/harnessCases/TemporaryTables.json +++ b/payloadbuilder-core/src/test/resources/harnessCases/TemporaryTables.json @@ -110,9 +110,10 @@ [{ "key": "newCol1", "value": 10 }, { "key": "newCol2", "value": 60 } ] ], [ - [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Hash Match" } ], - [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" } ], - [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Index Seek: \"#\".temp [newCol1] (ALL)" } ] + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" } ], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Hash Match" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Index Seek: \"#\".temp [newCol1] (ALL)" } ] ], [ [{ "key": "id", "value": 2 }, { "key": "row_id", "value": 1 }, { "key": "data", "value": "2_1" }, { "key": "newCol1", "value": 2 }, { "key": "newCol2", "value": 20 } ] @@ -144,9 +145,10 @@ "expectedResultSets": [ [], [ - [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Hash Match" } ], - [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" } ], - [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Index Seek: \"#\".temp [newCol1] (ALL)" } ] + [{ "key": "Node Id", "value": 3 }, { "key": "Parent Node Id", "value": null }, { "key": "Name", "value": "+- Write Output" } ], + [{ "key": "Node Id", "value": 2 }, { "key": "Parent Node Id", "value": 3 }, { "key": "Name", "value": "| +- Hash Match" } ], + [{ "key": "Node Id", "value": 0 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Scan: data" } ], + [{ "key": "Node Id", "value": 1 }, { "key": "Parent Node Id", "value": 2 }, { "key": "Name", "value": "| +- Index Seek: \"#\".temp [newCol1] (ALL)" } ] ], [] ]