Convert CTEs to subqueries at AST level for proper execution

Follow the stackql-devel pattern: CTEs are converted to Subqueries at AST
level in ast_expand.go, making downstream code treat them uniformly with
regular subqueries. Key changes:

- Change cteRegistry to store *sqlparser.Subquery instead of CommonTableExpr
- Add processCTEReference() which replaces TableName with Subquery in AST
- Handle CTE detection in AliasedTableExpr case (not TableName case)
- Remove CTEType-specific handling from from_rewrite.go, parameter_router.go,
  obtain_context.go, and dependencyplanner.go since SubqueryType path handles them
- Add logging for debugging CTE processing

Author: claude

internal/stackql/astanalysis/earlyanalysis/ast_expand.go

@@ -49,7 +49,7 @@ type indirectExpandAstVisitor struct {
 	selectCount                    int
 	mutateCount                    int
 	createBuilder                  []primitivebuilder.Builder
-	cteRegistry                    map[string]*sqlparser.CommonTableExpr
+	cteRegistry                    map[string]*sqlparser.Subquery // CTE name -> subquery definition
 }
 
 func newIndirectExpandAstVisitor(
@@ -76,7 +76,7 @@ func newIndirectExpandAstVisitor(
 		tcc:                 tcc,
 		whereParams:         whereParams,
 		indirectionDepth:    indirectionDepth,
-		cteRegistry:         make(map[string]*sqlparser.CommonTableExpr),
+		cteRegistry:         make(map[string]*sqlparser.Subquery),
 	}
 	return rv, nil
 }
@@ -105,6 +105,41 @@ func (v *indirectExpandAstVisitor) processMaterializedView(
 	return nil
 }
 
+// processCTEReference handles CTE references by converting them to subquery indirects.
+// Returns true if the table name was a CTE reference and was processed, false otherwise.
+func (v *indirectExpandAstVisitor) processCTEReference(
+	node *sqlparser.AliasedTableExpr,
+	tableName string,
+) bool {
+	cteSubquery, isCTE := v.cteRegistry[tableName]
+	if !isCTE {
+		return false
+	}
+	logging.GetLogger().Infof("processCTEReference: Converting CTE '%s' to subquery", tableName)
+	logging.GetLogger().Debugf("processCTEReference: CTE subquery = %s", sqlparser.String(cteSubquery))
+	// Modify the original node to replace the TableName with the CTE subquery
+	// This is critical: downstream code (GetHIDs) checks node.Expr type to identify subqueries
+	node.Expr = cteSubquery
+	// Set the alias to the CTE name if no explicit alias was provided
+	if node.As.IsEmpty() {
+		node.As = sqlparser.NewTableIdent(tableName)
+	}
+	logging.GetLogger().Debugf("processCTEReference: Node alias set to '%s'", node.As.GetRawVal())
+	sq := internaldto.NewSubqueryDTO(node, cteSubquery)
+	indirect, err := astindirect.NewSubqueryIndirect(sq)
+	if err != nil {
+		logging.GetLogger().Errorf("processCTEReference: Failed to create subquery indirect: %v", err)
+		return true //nolint:nilerr //TODO: investigate
+	}
+	err = v.processIndirect(node, indirect)
+	if err != nil {
+		logging.GetLogger().Errorf("processCTEReference: processIndirect failed: %v", err)
+	} else {
+		logging.GetLogger().Infof("processCTEReference: Successfully processed CTE '%s' as subquery", tableName)
+	}
+	return true
+}
+
 func (v *indirectExpandAstVisitor) processIndirect(node sqlparser.SQLNode, indirect astindirect.Indirect) error {
 	err := indirect.Parse()
 	if err != nil {
@@ -216,14 +251,14 @@ func (v *indirectExpandAstVisitor) Visit(node sqlparser.SQLNode) error {
 		addIf(node.StraightJoinHint, sqlparser.StraightJoinHint)
 		addIf(node.SQLCalcFoundRows, sqlparser.SQLCalcFoundRowsStr)
 
-		// Register CTEs (Common Table Expressions) if present.
-		// We only register CTE names here - the actual SELECT processing
-		// happens in processIndirect when the CTE is referenced.
-		// This mirrors how subqueries work (processed once in processIndirect).
+		// Extract CTEs from WITH clause and store in registry as Subqueries.
+		// CTEs are converted to subqueries at the AST level for uniform handling.
 		if node.With != nil {
+			logging.GetLogger().Infof("Registering %d CTEs from WITH clause", len(node.With.CTEs))
 			for _, cte := range node.With.CTEs {
 				cteName := cte.Name.GetRawVal()
-				v.cteRegistry[cteName] = cte
+				v.cteRegistry[cteName] = cte.Subquery
+				logging.GetLogger().Debugf("Registered CTE '%s' with subquery: %s", cteName, sqlparser.String(cte.Subquery))
 			}
 		}
 
@@ -798,6 +833,11 @@ func (v *indirectExpandAstVisitor) Visit(node sqlparser.SQLNode) error {
 					return nil //nolint:nilerr //TODO: investigate
 				}
 				return nil
+			case sqlparser.TableName:
+				// Check if this is a CTE reference - convert to subquery
+				if v.processCTEReference(node, n.GetRawVal()) {
+					return nil
+				}
 			}
 			err := node.Expr.Accept(v)
 			if err != nil {
@@ -835,22 +875,9 @@ func (v *indirectExpandAstVisitor) Visit(node sqlparser.SQLNode) error {
 		if node.IsEmpty() {
 			return nil
 		}
-		// Check if this is a CTE reference.
-		cteName := node.Name.GetRawVal()
-		if cte, isCTE := v.cteRegistry[cteName]; isCTE {
-			// This is a CTE reference - create and process an indirect.
-			// We need to call processIndirect to get the proper column context
-			// from the CTE's SELECT statement.
-			indirect, err := astindirect.NewCTEIndirect(cte)
-			if err != nil {
-				return err
-			}
-			err = v.processIndirect(node, indirect)
-			if err != nil {
-				return err
-			}
-			return nil
-		}
+		// Note: CTE references are handled in AliasedTableExpr case above,
+		// where they are converted to subqueries. This case only handles
+		// regular table names (provider.service.resource).
 		containsBackendMaterial := v.handlerCtx.GetDBMSInternalRouter().ExprIsRoutable(node)
 		if containsBackendMaterial {
 			v.containsNativeBackendMaterial = true

internal/stackql/astvisit/from_rewrite.go

@@ -668,15 +668,11 @@ func (v *standardFromRewriteAstVisitor) Visit(node sqlparser.SQLNode) error {
 					v.rewrittenQuery = templateString
 					v.indirectContexts = append(v.indirectContexts, indirect.GetSelectContext())
 				case astindirect.SubqueryType:
+					// Note: CTEs are converted to SubqueryType at AST level,
+					// so this path handles both regular subqueries and CTEs.
 					templateString := ` ( %s ) `
 					v.rewrittenQuery = templateString
 					v.indirectContexts = append(v.indirectContexts, indirect.GetSelectContext())
-				case astindirect.CTEType:
-					// CTEs are handled like views - the inner SELECT is executed
-					// and results are wrapped with the CTE name as alias.
-					templateString := fmt.Sprintf(` ( %%s ) AS "%s" `, name)
-					v.rewrittenQuery = templateString
-					v.indirectContexts = append(v.indirectContexts, indirect.GetSelectContext())
 				case astindirect.MaterializedViewType, astindirect.PhysicalTableType:
 					refString := fmt.Sprintf(` %s `, name)
 					isQuoted, _ := regexp.MatchString(`^".*"$`, name)

internal/stackql/dependencyplanner/dependencyplanner.go

@@ -164,10 +164,8 @@ func (dp *standardDependencyPlanner) Plan() error {
 			annotation := unit.GetAnnotation()
 			_, isView := annotation.GetView()
 			_, isSubquery := annotation.GetSubquery()
-			// Check if this is a CTE reference.
-			indirect, hasIndirect := annotation.GetTableMeta().GetIndirect()
-			isCTE := hasIndirect && indirect != nil && indirect.GetType() == astindirect.CTEType
-			if isView || isSubquery || isCTE {
+			// Note: CTEs are converted to subqueries at AST level, so isSubquery handles them.
+			if isView || isSubquery {
 				dp.annMap[tableExpr] = annotation
 				continue
 			}

internal/stackql/primitivegenerator/select.go

@@ -224,8 +224,8 @@ func (pb *standardPrimitiveGenerator) analyzeSelect(pbi planbuilderinput.PlanBui
 			selCtx := dp.GetSelectCtx()
 			pChild.GetPrimitiveComposer().SetBuilder(bld)
 			pb.PrimitiveComposer.SetSelectPreparedStatementCtx(selCtx)
-			// For indirects (like CTEs), also set the builder on the indirect's primitive
-			// composer so it can be found when parent's GetBuilder() iterates over indirects.
+			// For indirects (subqueries, views, etc.), also set the builder on the indirect's
+			// primitive composer so it can be found when parent's GetBuilder() iterates.
 			if pb.PrimitiveComposer.IsIndirect() {
 				pb.PrimitiveComposer.SetBuilder(bld)
 			}

internal/stackql/router/obtain_context/obtain_context.go

@@ -3,7 +3,6 @@ package obtain_context //nolint:revive,cyclop,stylecheck // TODO: allow
 import (
 	"fmt"
 
-	"github.com/stackql/stackql/internal/stackql/astindirect"
 	"github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto"
 	"github.com/stackql/stackql/internal/stackql/sql_system"
 	"github.com/stackql/stackql/internal/stackql/tablemetadata"
@@ -21,10 +20,8 @@ func ObtainAnnotationCtx(
 	_, isSQLDataSource := tbl.GetSQLDataSource()
 	_, isSubquery := tbl.GetSubquery()
 	isPGInternalObject := tbl.GetHeirarchyObjects().IsPGInternalObject()
-	// Check if this is a CTE reference.
-	indirect, hasIndirect := tbl.GetIndirect()
-	isCTE := hasIndirect && indirect != nil && indirect.GetType() == astindirect.CTEType
-	if isView || isSQLDataSource || isSubquery || isPGInternalObject || isCTE {
+	// Note: CTEs are converted to subqueries at AST level, so isSubquery handles them.
+	if isView || isSQLDataSource || isSubquery || isPGInternalObject {
 		// TODO: upgrade this flow; nil == YUCK!!!
 		return taxonomy.NewStaticStandardAnnotationCtx(
 			nil, tbl.GetHeirarchyObjects().GetHeirarchyIDs(),

internal/stackql/router/parameter_router.go

@@ -592,30 +592,22 @@ func (pr *standardParameterRouter) route(
 	priorParameters := runParamters.Clone()
 	// notOnStringParams := notOnParams.GetStringified()
 	// TODO: add parent params into the mix here.
-	// Check for CTE indirect before attempting hierarchy resolution.
-	// CTEs don't need provider/service/resource resolution.
-	cteIndirect, isCTE := pr.annotatedAST.GetIndirect(tb)
+	// Note: CTEs are converted to subqueries at AST level, so they follow
+	// the normal subquery path (handled via GetSubquery() in ObtainAnnotationCtx).
 	var hr tablemetadata.HeirarchyObjects
 	var err error
-	if isCTE && cteIndirect != nil && cteIndirect.GetType() == astindirect.CTEType {
-		// CTE reference - create empty hierarchy identifiers.
-		tableName := taxonomy.GetTableNameFromStatement(tb, pr.astFormatter)
-		hIDs := internaldto.NewHeirarchyIdentifiers("", "", tableName, "")
-		hr = tablemetadata.NewHeirarchyObjects(hIDs, isAwait)
+	hr, err = taxonomy.GetHeirarchyFromStatement(handlerCtx, tb, notOnParams, false, isAwait)
+	if err != nil {
+		hr, err = taxonomy.GetHeirarchyFromStatement(handlerCtx, tb, runParamters, false, isAwait)
 	} else {
-		hr, err = taxonomy.GetHeirarchyFromStatement(handlerCtx, tb, notOnParams, false, isAwait)
-		if err != nil {
-			hr, err = taxonomy.GetHeirarchyFromStatement(handlerCtx, tb, runParamters, false, isAwait)
-		} else {
-			// If the where parameters are sufficient, then need to switch
-			// the Table - Paramater coupling object
-			runParamters = notOnParams
-			priorParameters = priorNotOnParameters
-		}
-		// logging.GetLogger().Infof("hr = '%+v', remainingParams = '%+v', err = '%+v'", hr, remainingParams, err)
-		if err != nil {
-			return nil, err
-		}
+		// If the where parameters are sufficient, then need to switch
+		// the Table - Paramater coupling object
+		runParamters = notOnParams
+		priorParameters = priorNotOnParameters
+	}
+	// logging.GetLogger().Infof("hr = '%+v', remainingParams = '%+v', err = '%+v'", hr, remainingParams, err)
+	if err != nil {
+		return nil, err
 	}
 	// reconstitutedConsumedParams, err := tpc.ReconstituteConsumedParams(remainingParams)
 	// if err != nil {