Improved mock llm

src/engine/answerGenerator.ts

@@ -163,6 +163,12 @@ function extractReferencesFromResults(results: ToolResult[]): CopilotReferences
       extractLogReferences(result, refs.logs);
     }
 
+    // Extract team IDs
+    if (result.name.includes('team')) {
+      refs.teams = refs.teams || [];
+      extractTeamIds(result.result, refs.teams);
+    }
+
     // Extract orchestration plan IDs
     if (result.name.includes('orchestration')) {
       refs.orchestrationPlans = refs.orchestrationPlans || [];
@@ -176,6 +182,7 @@ function extractReferencesFromResults(results: ToolResult[]): CopilotReferences
   if (refs.alerts) refs.alerts = [...new Set(refs.alerts)];
   if (refs.deployments) refs.deployments = [...new Set(refs.deployments)];
   if (refs.tickets) refs.tickets = [...new Set(refs.tickets)];
+  if (refs.teams) refs.teams = [...new Set(refs.teams)];
   // Metrics and logs are complex objects, dedupe by JSON string
   if (refs.metrics) refs.metrics = dedupeByJson(refs.metrics);
   if (refs.logs) refs.logs = dedupeByJson(refs.logs);
@@ -326,6 +333,31 @@ function extractTicketIds(data: unknown, ids: string[]): void {
   }
 }
 
+function extractTeamIds(data: unknown, ids: string[]): void {
+  if (Array.isArray(data)) {
+    for (const item of data) {
+      if (typeof item === 'object' && item !== null) {
+        const obj = item as Record<string, unknown>;
+        if ('id' in obj) ids.push(String(obj.id));
+        else if ('name' in obj) ids.push(String(obj.name));
+      }
+    }
+  } else if (typeof data === 'object' && data !== null) {
+    const obj = data as Record<string, unknown>;
+    if ('id' in obj) ids.push(String(obj.id));
+    else if ('name' in obj) ids.push(String(obj.name));
+    if (Array.isArray(obj.teams)) {
+      for (const team of obj.teams) {
+        if (typeof team === 'object' && team !== null) {
+          const t = team as Record<string, unknown>;
+          if ('id' in t) ids.push(String(t.id));
+          else if ('name' in t) ids.push(String(t.name));
+        }
+      }
+    }
+  }
+}
+
 /**
  * Extract metric references from query-metrics tool results.
  * Uses the tool arguments to build a MetricReference with deep-linking metadata.

src/engine/capabilityRegistry.ts

@@ -168,6 +168,7 @@ function createFollowUpRegistry(): FollowUpRegistry {
 
   registry.register("query-logs", logFollowUpHandler);
 
+  registry.register("describe-metrics", metricFollowUpHandler);
   registry.register("query-metrics", metricFollowUpHandler);
 
   registry.register("query-tickets", ticketFollowUpHandler);
@@ -183,7 +184,7 @@ function createFollowUpRegistry(): FollowUpRegistry {
   registry.register("get-orchestration-plan", orchestrationFollowUpHandler);
 
   console.log(
-    "[FollowUpRegistry] Registered 17 follow-up handlers for 9 capabilities",
+    "[FollowUpRegistry] Registered 18 follow-up handlers for 9 capabilities",
   );
 
   return registry;

src/engine/copilotEngine.ts

@@ -324,6 +324,7 @@ export class CopilotEngine {
     const allExtractedEntities: Entity[] = [];
     let iteration = 0;
     let isFirstIteration = true;
+    let hadPlannedCalls = false;
 
     // Step 4: Reasoning Loop
     while (iteration < this.maxIterations) {
@@ -421,6 +422,11 @@ export class CopilotEngine {
       // Limit calls
       plannedCalls = this.limitToolCalls(plannedCalls);
 
+      // Track if any calls were ever planned (before toolRunner filtering)
+      if (plannedCalls.length > 0) {
+        hadPlannedCalls = true;
+      }
+
       // B. Check Stop Condition
       if (plannedCalls.length === 0) {
         console.log(
@@ -533,6 +539,14 @@ export class CopilotEngine {
       this.config.llm,
     );
 
+    // If calls were planned but no results were collected, tool calls were likely
+    // skipped due to unresolved placeholder arguments (e.g. {{incidentId}})
+    if (hadPlannedCalls && allResults.length === 0) {
+      answer.missing = answer.missing
+        ? [...answer.missing, 'tool outputs']
+        : ['tool outputs'];
+    }
+
     // Step 6: Create TurnExecutionTrace from ExecutionTrace
     const turnTrace: TurnExecutionTrace = {
       traceId: trace.traceId,

src/engine/followUpEngine.ts

@@ -29,6 +29,7 @@ export class FollowUpEngine {
           chatId,
           conversationHistory,
           userQuestion,
+          toolResults,
         );
 
         try {
@@ -66,12 +67,13 @@ export class FollowUpEngine {
     chatId: string,
     conversationHistory: ConversationTurn[],
     userQuestion: string,
+    currentResults: ToolResult[],
   ): HandlerContext {
     return {
       chatId,
       turnNumber: conversationHistory.length,
       conversationHistory,
-      toolResults: [result],
+      toolResults: currentResults,
       userQuestion,
     };
   }

src/engine/handlers/incident/scopeHandler.ts

@@ -156,6 +156,26 @@ export const incidentScopeInferenceHandler: ScopeHandler = async (
 
   // Note: We no longer look at conversation history toolResults since they're not stored.
   // Scope from previous turns should be inferred from entities or the current turn's results.
+  if (!hasScope || !(scope.service && scope.environment && scope.team)) {
+    for (let i = context.conversationHistory.length - 1; i >= 0; i--) {
+      const turn = context.conversationHistory[i];
+      if (turn.entities) {
+        for (const entity of turn.entities) {
+          if (entity.type === "service" && !scope.service) {
+            scope.service = entity.value;
+            hasScope = true;
+          }
+          if (entity.type === "team" && !scope.team) {
+            scope.team = entity.value;
+            hasScope = true;
+          }
+        }
+      }
+      if (scope.service && scope.environment && scope.team) {
+        break;
+      }
+    }
+  }
 
   return hasScope ? scope : null;
 };

src/engine/handlers/metric/followUpHandler.ts

@@ -9,7 +9,7 @@
  */
 
 import type { FollowUpHandler } from "../handlers.js";
-import type { ToolCall, JsonObject } from "../../../types.js";
+import type { ToolCall, ToolResult, JsonObject } from "../../../types.js";
 import { generateSearchExpression } from "../logQueryParser.js";
 import { HandlerUtils } from "../utils.js";
 
@@ -29,12 +29,170 @@
   "timeout",
 ];
 
+function extractMetricNames(result: ToolResult["result"]): string[] {
+  if (Array.isArray(result)) {
+    return result
+      .map((entry) => {
+        if (typeof entry === "string") return entry;
+        if (entry && typeof entry === "object" && "name" in entry) {
+          const name = (entry as JsonObject).name;
+          return typeof name === "string" ? name : undefined;
+        }
+        return undefined;
+      })
+      .filter((name): name is string => typeof name === "string" && name.length > 0);
+  }
+
+  if (result && typeof result === "object") {
+    const metrics = (result as JsonObject).metrics;
+    if (Array.isArray(metrics)) {
+      return extractMetricNames(metrics);
+    }
+  }
+
+  return [];
+}
+
+function shouldQueryDiscoveredMetrics(question: string): boolean {
+  const lower = question.toLowerCase();
+  const discoveryOnlyPatterns = [
+    "available metrics",
+    "list metrics",
+    "what metrics",
+    "which metrics",
+  ];
+  if (discoveryOnlyPatterns.some((pattern) => lower.includes(pattern))) {
+    return false;
+  }
+
+  const investigationTerms = [
+    "metric",
+    "metrics",
+    "cpu",
+    "memory",
+    "latency",
+    "p95",
+    "p99",
+    "throughput",
+    "request",
+    "error",
+  ];
+  const actionTerms = [
+    "check",
+    "show",
+    "inspect",
+    "analy",
+    "graph",
+    "trend",
+    "root cause",
+    "why",
+    "high",
+  ];
+
+  return (
+    investigationTerms.some((term) => lower.includes(term)) &&
+    actionTerms.some((term) => lower.includes(term))
+  );
+}
+
+function selectMetricName(question: string, metricNames: string[]): string | undefined {
+  const lower = question.toLowerCase();
+  const preferredTokens = [
+    "cpu",
+    "memory",
+    "latency",
+    "p99",
+    "p95",
+    "throughput",
+    "request",
+    "error",
+  ];
+
+  for (const token of preferredTokens) {
+    if (!lower.includes(token)) continue;
+    const match = metricNames.find((name) => name.toLowerCase().includes(token));
+    if (match) return match;
+  }
+
+  return metricNames[0];
+}
+
+function getIncidentTimeWindow(context: Parameters<FollowUpHandler>[0]): {
+  start: string;
+  end: string;
+} | null {
+  for (const result of context.toolResults) {
+    if (result.name !== "query-incidents" && result.name !== "get-incident") {
+      continue;
+    }
+
+    const incident = Array.isArray(result.result)
+      ? result.result[0]
+      : result.result;
+    if (!incident || typeof incident !== "object") {
+      continue;
+    }
+
+    const incidentObject = incident as JsonObject;
+    const startValue = incidentObject.startTime ?? incidentObject.createdAt;
+    const endValue = incidentObject.endTime ?? incidentObject.updatedAt;
+    if (typeof startValue !== "string" && typeof endValue !== "string") {
+      continue;
+    }
+
+    const expanded = HandlerUtils.expandTimeWindow(
+      typeof startValue === "string" ? startValue : undefined,
+      typeof endValue === "string" ? endValue : undefined,
+      15,
+    );
+    return {
+      start: expanded.start.toISOString(),
+      end: expanded.end.toISOString(),
+    };
+  }
+
+  return null;
+}
+
 export const metricFollowUpHandler: FollowUpHandler = async (
   context,
   toolResult,
 ): Promise<ToolCall[]> => {
   const suggestions: ToolCall[] = [];
 
+  if (toolResult.name === "describe-metrics") {
+    const metricNames = extractMetricNames(toolResult.result);
+    const scope = toolResult.arguments?.scope as JsonObject | undefined;
+    const service = scope?.service;
+
+    if (
+      typeof service === "string" &&
+      metricNames.length > 0 &&
+      shouldQueryDiscoveredMetrics(context.userQuestion) &&
+      !HandlerUtils.isDuplicateToolCall(context, "query-metrics", service)
+    ) {
+      const metricName = selectMetricName(context.userQuestion, metricNames);
+      if (metricName) {
+        const incidentWindow = getIncidentTimeWindow(context);
+        const end = incidentWindow?.end ?? new Date().toISOString();
+        const start = incidentWindow?.start ?? new Date(Date.now() - 60 * 60 * 1000).toISOString();
+
+        suggestions.push({
+          name: "query-metrics",
+          arguments: {
+            scope: { service },
+            expression: { metricName },
+            step: 60,
+            start,
+            end,
+          },
+        });
+      }
+    }
+
+    return suggestions;
+  }
+
   if (!toolResult.result || typeof toolResult.result !== "object") {
     return suggestions;
   }
@@ -139,7 +297,7 @@
       // Check if we already added it to suggestions in this turn
       const alreadySuggested = suggestions.some(s =>
         s.name === "describe-metrics" &&
         (s.arguments.scope as any)?.service === service
       );
 
       if (!alreadySuggested) {
@@ -155,4 +313,3 @@
 
   return suggestions;
 };
-

src/engine/handlers/metric/queryBuilder.ts

@@ -20,7 +20,7 @@ import { QueryBuilderHandler } from "../handlers.js";
 import { JsonObject } from "../../../types.js";
 
 export const metricQueryBuilder: QueryBuilderHandler = async (
-    _context,
+    context,
     _toolName,
     naturalLanguage,
 ): Promise<JsonObject> => {
@@ -42,7 +42,30 @@ export const metricQueryBuilder: QueryBuilderHandler = async (
     } else if (lower.includes("request") || lower.includes("throughput")) {
         expression.metricName = "requests";
     }
-    // Note: We don't guess names like "error_rate" that may not exist
+
+    // If no hint from natural language, try to use a discovered metric from describe-metrics results
+    if (!expression.metricName) {
+        for (const result of context.toolResults) {
+            if (result.name === "describe-metrics") {
+                const data = result.result;
+                const list = Array.isArray(data) ? data
+                    : (data && typeof data === "object" && Array.isArray((data as Record<string, unknown>).metrics))
+                        ? (data as Record<string, unknown>).metrics as unknown[]
+                        : null;
+                if (list && list.length > 0) {
+                    const first = list[0];
+                    const name = typeof first === "string" ? first
+                        : (first && typeof first === "object" && "name" in (first as object))
+                            ? String((first as Record<string, unknown>).name)
+                            : undefined;
+                    if (name) {
+                        expression.metricName = name;
+                        break;
+                    }
+                }
+            }
+        }
+    }
 
     // MCP schema: start/end must be ISO 8601 datetime
     const now = new Date();