Fixed the runtime inconsistencies in the copilot loop with tool resul…

package.json

@@ -30,7 +30,7 @@
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "type-check": "tsc --noEmit",
-    "test": "NODE_ENV=test node --test --loader ts-node/esm",
+    "test": "NODE_ENV=test node --test --loader ts-node/esm tests/*.test.ts tests/**/*.test.ts",
     "start": "node --loader ts-node/esm src/server.ts",
     "seed": "node --loader ts-node/esm scripts/seedDatabase.ts"
   },

src/engine/chatNamer.ts

@@ -73,6 +73,9 @@ export class ChatNamer {
     // Determine intent from user message
     const intent = this.determineIntent(userMessage);
 
+    // Avoid redundant topic/service naming like "Payment Payment Issues"
+    this.deduplicateSemanticOverlap(mergedEntities);
+
     // Try to synthesize a name
     let name = this.synthesizeName(mergedEntities, intent, userMessage);
 
@@ -96,6 +99,30 @@ export class ChatNamer {
     return sanitizedName;
   }
 
+  private deduplicateSemanticOverlap(entities: ExtractedEntities): void {
+    if (entities.services.length === 0 || entities.topics.length === 0) {
+      return;
+    }
+
+    const topicSet = new Set(entities.topics.map((topic) => topic.toLowerCase()));
+    entities.services = entities.services.filter((service) => {
+      const normalizedService = service.toLowerCase();
+      for (const topic of topicSet) {
+        if (
+          normalizedService === topic ||
+          normalizedService.startsWith(`${topic}-`) ||
+          normalizedService.endsWith(`-${topic}`) ||
+          normalizedService.includes(`${topic}-service`) ||
+          normalizedService.includes(`${topic}-svc`) ||
+          normalizedService.includes(`${topic}-api`)
+        ) {
+          return false;
+        }
+      }
+      return true;
+    });
+  }
+
   /**
    * Extract key entities from the LLM response (incidents, services, metrics).
    */
@@ -359,10 +386,10 @@ export class ChatNamer {
       return `${topic} ${metric} Issues`;
     }
 
-    // Priority 6: Topic + Problems/Issues (e.g., "Payment Service Issues")
+    // Priority 6: Topic + Problems/Issues (e.g., "Payment Issues")
     if (topics.length > 0 && /problem|issue|error|fail/i.test(userMessage)) {
       const topic = this.formatTopicName(topics[0]);
-      return `${topic} Service Issues`;
+      return `${topic} Issues`;
     }
 
     // Priority 7: Topic + Intent (e.g., "Payment Investigation")

src/engine/conversationManager.ts

@@ -11,6 +11,7 @@ import {
   Entity,
   ConversationContext,
   TurnExecutionTrace,
+  ToolResult,
 } from "../types.js";
 import { ConversationStore } from "../conversationStore.js";
 import { createConversationStore } from "../storeFactory.js";
@@ -99,6 +100,7 @@ export class ConversationManager {
     assistantResponse?: string,
     entities?: Entity[],
     executionTrace?: TurnExecutionTrace,
+    toolResults?: ToolResult[],
   ): Promise<void> {
     let conversation = await this.store.get(chatId);
 
@@ -119,6 +121,7 @@ export class ConversationManager {
       assistantResponse,
       timestamp: Date.now(),
       entities,
+      toolResults,
       executionTrace,
     });
 
@@ -135,8 +138,8 @@ export class ConversationManager {
 
   /**
    * Build LLM message history from conversation turns.
-   * Note: Tool results are no longer stored in turns (replaced by executionTrace).
-   * The message history now only includes user messages and assistant responses.
+   * Tool results are stored alongside turns so follow-up planning can reuse
+   * concrete outputs instead of relying only on assistant summaries.
    */
   async buildMessageHistory(chatId: string): Promise<LlmMessage[]> {
     const conversation = await this.getConversation(chatId);
@@ -153,6 +156,14 @@ export class ConversationManager {
         content: turn.userMessage,
       });
 
+      for (const toolResult of turn.toolResults ?? []) {
+        messages.push({
+          role: "tool",
+          toolName: toolResult.name,
+          content: summarizeToolResult(toolResult),
+        });
+      }
+
       // Add assistant response if any
       if (turn.assistantResponse) {
         messages.push({
@@ -294,3 +305,21 @@ export class ConversationManager {
     return this.store;
   }
 }
+
+function summarizeToolResult(toolResult: ToolResult): string {
+  const content = safeStringify(toolResult.result);
+  const suffix = content.length > 1000 ? `${content.slice(0, 1000)}...` : content;
+  return `${toolResult.name}: ${suffix}`;
+}
+
+function safeStringify(value: unknown): string {
+  if (typeof value === "string") {
+    return value;
+  }
+
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return String(value);
+  }
+}

src/engine/copilotEngine.ts

@@ -155,7 +155,7 @@ export class CopilotEngine {
 
     // Step 1: Check cache for each call
     for (const call of calls) {
-      const cached = this.resultCache.get(call);
+      const cached = this.resultCache.get(call, chatId);
       if (cached) {
         console.log(
           `[Copilot][${chatId}] Using cached result for ${call.name}`,
@@ -225,6 +225,10 @@ export class CopilotEngine {
       for (let i = 0; i < callsToExecute.length; i++) {
         const result = freshResults[i];
         if (!result) continue; // Skip if somehow undefined
+        const executedCall: ToolCall = {
+          ...callsToExecute[i],
+          arguments: result.arguments ?? callsToExecute[i].arguments,
+        };
 
         const isError =
           typeof result.result === "object" &&
@@ -233,14 +237,14 @@ export class CopilotEngine {
 
         // Only cache if not an error
         if (!isError) {
-          this.resultCache.set(callsToExecute[i], result);
+          this.resultCache.set(executedCall, result, chatId);
         }
 
         // Record tool execution in trace
         if (trace) {
           this.executionTracer.recordToolExecution(trace, {
-            toolName: callsToExecute[i].name,
-            arguments: callsToExecute[i].arguments,
+            toolName: executedCall.name,
+            arguments: executedCall.arguments,
             cacheHit: false,
             executionTimeMs: Math.round(executionTime / callsToExecute.length), // Approximate per-tool time
             success: !isError,
@@ -304,6 +308,7 @@ export class CopilotEngine {
     const resolutions = await this.referenceResolver.resolveReferences(
       question,
       entityContext,
+      conversationTurns,
     );
     const resolvedQuestion = this.referenceResolver.applyResolutions(
       question,
@@ -421,6 +426,7 @@ export class CopilotEngine {
 
       // Limit calls
       plannedCalls = this.limitToolCalls(plannedCalls);
+      this.executionTracer.updateIterationPlan(trace, plannedCalls);
 
       // Track if any calls were ever planned (before toolRunner filtering)
       if (plannedCalls.length > 0) {
@@ -563,6 +569,7 @@ export class CopilotEngine {
       answer.conclusion,
       allExtractedEntities,
       turnTrace,
+      allResults,
     );
 
     // Step 8: Generate conversation name for new conversations only

src/engine/executionTracer.ts

@@ -39,6 +39,16 @@ export class ExecutionTracer {
     trace.iterations.push(iteration);
   }
 
+  /**
+   * Update the current iteration plan after heuristics/refinement.
+   */
+  updateIterationPlan(trace: ExecutionTrace, plannedTools: ToolCall[]): void {
+    const currentIteration = this.getCurrentIteration(trace);
+    if (currentIteration) {
+      currentIteration.plannedTools = [...plannedTools];
+    }
+  }
+
   /**
    * Record a heuristic modification
    */

src/engine/handlers/service/entityHandler.ts

@@ -26,6 +26,15 @@ export const serviceEntityHandler: EntityHandler = async (
   // Handle array of services (query-services returns z.array(serviceSchema))
   if (Array.isArray(toolResult.result)) {
     services = toolResult.result as JsonObject[];
+  } else if (
+    typeof toolResult.result === "object" &&
+    toolResult.result !== null &&
+    Array.isArray((toolResult.result as JsonObject).services)
+  ) {
+    services = ((toolResult.result as JsonObject).services as Array<JsonObject | string>).map(
+      (service) =>
+        typeof service === "string" ? ({ name: service } as JsonObject) : service,
+    );
   } else {
     // Single service
     services = [toolResult.result as JsonObject];

src/engine/handlers/service/referenceHandler.ts

@@ -42,6 +42,15 @@ export const serviceReferenceHandler: ReferenceHandler = async (
         let servicesArray: Array<Record<string, unknown>> = [];
         if (Array.isArray(content)) {
           servicesArray = content as Array<Record<string, unknown>>;
+        } else if (
+          typeof content === "object" &&
+          content !== null &&
+          Array.isArray((content as Record<string, unknown>).services)
+        ) {
+          servicesArray = (content as { services: Array<Record<string, unknown> | string> }).services
+            .map((service) =>
+              typeof service === "string" ? { name: service } : service,
+            );
         } else if (typeof content === "object" && content !== null) {
           servicesArray = [content as Record<string, unknown>];
         }

src/engine/parallelToolRunner.ts

@@ -74,7 +74,7 @@ export class ParallelToolRunner {
     );
 
     for (let i = 0; i < batches.length; i++) {
-      const batch = batches[i];
+      const batch = this.resolveBatchDependencies(batches[i], results);
       console.log(
         `[ParallelToolRunner][${logId}] Batch ${i + 1}/${batches.length}: ${batch.map((c) => c.name).join(", ")}`,
       );
@@ -87,6 +87,92 @@ export class ParallelToolRunner {
     return results;
   }
 
+  private resolveBatchDependencies(
+    batch: ToolCall[],
+    previousResults: ToolResult[],
+  ): ToolCall[] {
+    return batch.map((call) => {
+      if (call.name === "get-incident-timeline" && !call.arguments?.id) {
+        const incidentId = this.findFirstId(previousResults, "query-incidents");
+        if (incidentId) {
+          return {
+            ...call,
+            arguments: {
+              ...call.arguments,
+              id: incidentId,
+            },
+          };
+        }
+      }
+
+      if (call.name === "get-ticket" && !call.arguments?.id) {
+        const ticketId = this.findFirstId(previousResults, "query-tickets");
+        if (ticketId) {
+          return {
+            ...call,
+            arguments: {
+              ...call.arguments,
+              id: ticketId,
+            },
+          };
+        }
+      }
+
+      return call;
+    });
+  }
+
+  private findFirstId(
+    results: ToolResult[],
+    sourceToolName: string,
+  ): string | undefined {
+    for (const result of results) {
+      if (result.name !== sourceToolName) continue;
+
+      const extracted = this.extractIdFromResult(result.result);
+      if (extracted) {
+        return extracted;
+      }
+    }
+
+    return undefined;
+  }
+
+  private extractIdFromResult(result: ToolResult["result"]): string | undefined {
+    if (Array.isArray(result)) {
+      for (const item of result) {
+        if (item && typeof item === "object" && !Array.isArray(item)) {
+          const maybeId = (item as Record<string, unknown>).id;
+          if (typeof maybeId === "string" && maybeId.trim()) {
+            return maybeId;
+          }
+        }
+      }
+      return undefined;
+    }
+
+    if (result && typeof result === "object") {
+      const record = result as Record<string, unknown>;
+      if (typeof record.id === "string" && record.id.trim()) {
+        return record.id;
+      }
+
+      const nestedArrays = Object.values(record).filter(Array.isArray) as unknown[][];
+      for (const arrayValue of nestedArrays) {
+        for (const item of arrayValue) {
+          if (item && typeof item === "object" && !Array.isArray(item)) {
+            const maybeId = (item as Record<string, unknown>).id;
+            if (typeof maybeId === "string" && maybeId.trim()) {
+              return maybeId;
+            }
+          }
+        }
+      }
+    }
+
+    return undefined;
+  }
+
   /**
    * Check if tools can be executed in parallel (no dependencies)
    */

src/engine/planRefiner.ts

@@ -87,8 +87,14 @@ export class PlanRefiner {
         }
       } else if (validation.replacementCall) {
         // Validation failed but a replacement was suggested
-        console.log(`[PlanRefiner] Replacing ${call.name} with ${validation.replacementCall.name}`);
-        replacementCalls.push(validation.replacementCall);
+        if (tools.some((candidate) => candidate.name === validation.replacementCall?.name)) {
+          console.log(`[PlanRefiner] Replacing ${call.name} with ${validation.replacementCall.name}`);
+          replacementCalls.push(validation.replacementCall);
+        } else {
+          console.log(
+            `[PlanRefiner] Skipping replacement ${validation.replacementCall.name} for ${call.name} because the tool is unavailable`,
+          );
+        }
         validatedCalls.push({ call, valid: false });
       } else {
         // Validation failed with no replacement
@@ -183,4 +189,3 @@ export class PlanRefiner {
     return augmented;
   }
 }
-