Dynamic query plans

payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/AQueryResultImpl.java

@@ -8,6 +8,7 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.time.StopWatch;
 
 import se.kuseman.payloadbuilder.api.OutputWriter;
@@ -25,6 +26,11 @@
 import se.kuseman.payloadbuilder.core.execution.OutputWriterUtils;
 import se.kuseman.payloadbuilder.core.execution.QuerySession;
 import se.kuseman.payloadbuilder.core.execution.StatementContext;
+import se.kuseman.payloadbuilder.core.parser.Location;
+import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor;
+import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnalyzeData;
+import se.kuseman.payloadbuilder.core.physicalplan.AnalyzeVisitor.AnlayzeType;
+import se.kuseman.payloadbuilder.core.physicalplan.DescribePlan;
 import se.kuseman.payloadbuilder.core.physicalplan.IPhysicalPlan;
 import se.kuseman.payloadbuilder.core.statement.CacheFlushRemoveStatement;
 import se.kuseman.payloadbuilder.core.statement.DescribeSelectStatement;
@@ -166,6 +172,27 @@ public Void visit(StatementList statement, Void context)
     public Void visit(PhysicalStatement statement, Void ctx)
     {
         currentPlan = statement.getPlan();
+
+        if (!(currentPlan instanceof DescribePlan))
+        {
+            AnalyzeData analyzeData = AnalyzeVisitor.AnalyzeData.fromSession(session);
+            if (analyzeData.type() != AnlayzeType.NONE)
+            {
+                String queryText = ObjectUtils.getIfNull(statement.getLocation(), Location.EMPTY)
+                        .text();
+                IPhysicalPlan analyzePlan = AnalyzeVisitor.describe(currentPlan, analyzeData.type(), analyzeData.format(), queryText);
+                if (analyzeData.extendOutput())
+                {
+                    // Run currentPlan as usual and the add the analyze on top on queue
+                    queue.add(0, new PhysicalStatement(analyzePlan, null));
+                }
+                else
+                {
+                    currentPlan = analyzePlan;
+                }
+            }
+        }
+
         return null;
     }
 
@@ -222,7 +249,7 @@ protected void processCurrentPlan(OutputWriter writer, ResultConsumer resultCons
         Schema schema = currentPlan.getSchema();
         boolean currentPlanIsAsterisk = SchemaUtils.isAsterisk(schema);
         boolean currentPlanHasAsteriskInput = currentPlanIsAsterisk
-                || SchemaUtils.isAsterisk(schema, true);
+                || SchemaUtils.originatesFromAsteriskInput(schema);
         if (resultConsumer != null)
         {
             resultConsumer.schema(currentPlanIsAsterisk ? Schema.EMPTY

payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/common/SchemaUtils.java

@@ -94,14 +94,22 @@ else if (schema2 == null
         return new Schema(columns);
     }
 
-    /** Returns true if this schema contains asterisk columns */
+    /**
+     * Returns true if provided schema has columns that originates from any asterisk input. Ie. any column on any level that is of type {@link CoreColumn.Type#ASTERISK} or
+     * {@link CoreColumn.Type#NAMED_ASTERISK}
+     */
+    public static boolean originatesFromAsteriskInput(Schema schema)
+    {
+        return isAsterisk(schema, true, true);
+    }
+
+    /** Returns true if this schema contains asterisk columns. One or more columns of type {@link CoreColumn.Type#ASTERISK}. */
     public static boolean isAsterisk(Schema schema)
     {
-        return isAsterisk(schema, false);
+        return isAsterisk(schema, false, false);
     }
 
-    /** Returns true if this schema contains asterisk columns */
-    public static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks)
+    private static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks, boolean traverseIntoSubSchemas)
     {
         int size = schema.getSize();
         if (size == 0)
@@ -113,7 +121,7 @@ public static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks)
         {
             Column column = schema.getColumns()
                     .get(i);
-            if (isAsterisk(column, includeNamedAsterisks))
+            if (isAsterisk(column, includeNamedAsterisks, traverseIntoSubSchemas))
             {
                 return true;
             }
@@ -126,13 +134,10 @@ public static boolean isAsterisk(Schema schema, boolean includeNamedAsterisks)
      */
     public static boolean isAsterisk(Column column)
     {
-        return isAsterisk(column, false);
+        return isAsterisk(column, false, false);
     }
 
-    /**
-     * Returns true if provided column is asterisk.
-     */
-    public static boolean isAsterisk(Column column, boolean includeNamedAsterisks)
+    private static boolean isAsterisk(Column column, boolean includeNamedAsterisks, boolean traverseIntoSubSchemas)
     {
         if (column instanceof CoreColumn cc)
         {
@@ -144,11 +149,15 @@ public static boolean isAsterisk(Column column, boolean includeNamedAsterisks)
             }
 
             // Dig down into complex type schema
-            if (cc.getType()
-                    .getSchema() != null)
+            // We also do this for populated columns as those count as asterisk
+            // if their column is asterisk
+            if ((traverseIntoSubSchemas
+                    || cc.getColumnType() == CoreColumn.Type.POPULATED)
+                    && cc.getType()
+                            .getSchema() != null)
             {
                 return isAsterisk(cc.getType()
-                        .getSchema(), includeNamedAsterisks);
+                        .getSchema(), includeNamedAsterisks, traverseIntoSubSchemas);
             }
         }
         return false;

payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/execution/QuerySession.java

@@ -55,17 +55,26 @@
  **/
 public class QuerySession implements IQuerySession
 {
-    /* System properties */
+    /* Compile time System properties */
     /** Enable to print logical and physical plans to session print writer */
     public static final String PRINT_PLAN = "printplan";
     /** Enable to print all logical plans for each optimisation rule. */
     public static final String DEBUG_PLAN = "debugplan";
-
     /** Force a nested loop where default would have been a hash match */
     public static final String FORCE_NESTED_LOOP = "force_nested_loop";
+    // ** Force no inner cache for non correlated nested loops */
+    // public static final String FORCE_NO_INNER_CACHE = "force_no_inner_cache";
+
+    /* Runtime System properties */
+    /** Enable describe of all plans without explicit DESCRIBE key word. This dynamically injects a describe plan on top of all plans. */
+    public static final String PLAN_DESCRIBE = "plan_describe";
+    /** Enable analyze of all plans without explicit ANALYZE key word. This dynamically injects a analyze plan on top of all plans. */
+    public static final String PLAN_ANALYZE = "plan_analyze";
+    /** Format of plan output. Only applicable {@link #PLAN_ANALYZE}, {@link #PLAN_DESCRIBE} or explicit DESCRIBE/ANALYZE keyword is set. */
+    public static final String PLAN_FORMAT = "plan_format";
+    /** Flag that indicates that the query result should be returned along with the plan. Only applicable {@link #PLAN_ANALYZE}, {@link #PLAN_DESCRIBE} or explicit DESCRIBE/ANALYZE keyword is set. */
+    public static final String PLAN_EXTENDED_OUTPUT = "plan_extended_output";
 
-    /** Force no inner cache for non correlated nested loops */
-    public static final String FORCE_NO_INNER_CACHE = "force_no_inner_cache";
     /* End system properties */
 
     /* Compile fields */

payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/logicalplan/optimization/ColumnResolver.java

@@ -699,19 +699,9 @@ public ILogicalPlan visit(OperatorFunctionScan plan, Ctx context)
         ResolvedType type = pair.getValue()
                 .getType(input.getSchema());
 
-        // If the function type has a sub schema which is asterisk then we treat this column as asterisk
-        // to easier detect if a schema is asterisk without the need to dig into the sub schema
-        CoreColumn.Type columnType = CoreColumn.Type.REGULAR;
-        if (type.getSchema() != null
-                && SchemaUtils.isAsterisk(type.getSchema(), true))
-        {
-            columnType = CoreColumn.Type.ASTERISK;
-        }
-
         // Recreate the operator schema
         // Use type from the resolved input and name from the plans schema
         Schema schema = Schema.of(CoreColumn.Builder.from(name, type)
-                .withColumnType(columnType)
                 .withInternal(isInternal)
                 .build());
 
@@ -1372,15 +1362,6 @@ private ResolveResult resolve(ColumnResolver.Ctx context, QualifiedName original
                 // We only look for direct asterisk columns here
                 boolean isAsterisk = SchemaUtils.getColumnType(schemaColumn) == CoreColumn.Type.ASTERISK;
 
-                // We treat internal columns as non asterisk when resolving even if their schema is asterisk due
-                // to it's input.
-                // This happens when a subquery expression is pushed down to a join and is getting a generated name (__expr0 etc.)
-                // and we hit that column, if that one is asterisk we must treat it like a non asterisk match
-                if (SchemaUtils.isInternal(schemaColumn))
-                {
-                    isAsterisk = false;
-                }
-
                 String schemaAlias = p.getKey();
                 // If the schema doesn't have an alias check to see if the column has a TableSourceReference
                 // attached and if so use the as schemaAlias

payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/Location.java

@@ -1,18 +1,26 @@
 package se.kuseman.payloadbuilder.core.parser;
 
+import static java.util.Objects.requireNonNull;
+
 import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.misc.Interval;
 import org.antlr.v4.runtime.tree.TerminalNode;
 
 /** A location in query text */
-public record Location(int line, int startOffset, int endOffset)
+public record Location(int line, int startOffset, int endOffset, String text)
 {
 
-    public static Location EMPTY = new Location(0, 0, 0);
+    public static Location EMPTY = new Location(0, 0, 0, "");
+
+    public Location
+    {
+        requireNonNull(text);
+    }
 
     /** Create a location from provided rule context */
     public static Location from(ParserRuleContext ctx)
     {
-        return new Location(ctx.start.getLine(), ctx.start.getStartIndex(), ctx.stop.getStopIndex() + 1);
+        return new Location(ctx.start.getLine(), ctx.start.getStartIndex(), ctx.stop.getStopIndex() + 1, "");
     }
 
     /** Create a location from provided rule context */
@@ -23,6 +31,17 @@ public static Location from(TerminalNode node)
                 node.getSymbol()
                         .getStartIndex(),
                 node.getSymbol()
-                        .getStopIndex() + 1);
+                        .getStopIndex() + 1,
+                "");
     }
+
+    /** Create a location from provided rule context extracting the rule text. */
+    public static Location withText(ParserRuleContext ctx)
+    {
+        String text = ctx.start.getInputStream()
+                .getText(new Interval(ctx.start.getStartIndex(), ctx.stop.getStopIndex()));
+
+        return new Location(ctx.start.getLine(), ctx.start.getStartIndex(), ctx.stop.getStopIndex() + 1, text);
+    }
+
 }

payloadbuilder-core/src/main/java/se/kuseman/payloadbuilder/core/parser/QueryParser.java

@@ -228,7 +228,7 @@ else if (e instanceof LexerNoViableAltException)
                     }
                 }
 
-                Location location = new Location(line, startIndex, stopIndex);
+                Location location = new Location(line, startIndex, stopIndex, "");
                 throw new ParseException(msg, location);
             }
         };
@@ -469,10 +469,10 @@ public Object visitInsertStatement(InsertStatementContext ctx)
                     inputInput = limit.getInput();
                 }
 
-                input = new LogicalSelectStatement(new Limit(inputInput, topCount), false);
+                input = new LogicalSelectStatement(new Limit(inputInput, topCount), false, Location.withText(ctx));
             }
             TableName tableName = tableName(ctx.tableName(), ctx.intoOptions);
-            return new LogicalInsertIntoStatement(input, tableName.catalogAlias, tableName.table, columns, tableName.options, Location.from(ctx));
+            return new LogicalInsertIntoStatement(input, tableName.catalogAlias, tableName.table, columns, tableName.options, Location.withText(ctx));
         }
 
         @Override
@@ -490,7 +490,7 @@ public Object visitInsertStatementValue(InsertStatementValueContext ctx)
                     .toList();
 
             ConstantScan constantScan = ConstantScan.createFromRows(expressions, Location.from(ctx.tableValueConstructor()));
-            return new LogicalSelectStatement(constantScan, false);
+            return new LogicalSelectStatement(constantScan, false, Location.withText(ctx));
         }
 
         @Override
@@ -531,7 +531,7 @@ public Object visitSelectStatement(SelectStatementContext ctx)
             plan = wrapTop(plan, ctx);
             plan = wrapOperatorFunction(plan, ctx);
 
-            Statement statement = new LogicalSelectStatement(plan, assignmentSelect);
+            Statement statement = new LogicalSelectStatement(plan, assignmentSelect, Location.withText(ctx));
 
             if (ctx.into != null)
             {
@@ -541,7 +541,7 @@ public Object visitSelectStatement(SelectStatementContext ctx)
                 }
 
                 TableName tableName = tableName(ctx.into, ctx.intoOptions);
-                statement = new LogicalSelectIntoStatement((LogicalSelectStatement) statement, tableName.catalogAlias, tableName.table, tableName.options, Location.from(ctx.into));
+                statement = new LogicalSelectIntoStatement((LogicalSelectStatement) statement, tableName.catalogAlias, tableName.table, tableName.options, Location.withText(ctx));
             }
 
             return statement;