diff --git a/cli/commands/batch-evaluate-command.ts b/cli/commands/batch-evaluate-command.ts index 30fec87..db0e7ca 100644 --- a/cli/commands/batch-evaluate-command.ts +++ b/cli/commands/batch-evaluate-command.ts @@ -9,6 +9,7 @@ import { createEvaluationDirectory, EvaluationMetadata, printBatchCompletionMessage, + getEvaluationRoot, } from '../utils/shared.utils'; import { ProgressTracker } from '../utils/progress-tracker'; import { CostEstimatorService } from '../../src/services/cost-estimator.service'; @@ -17,6 +18,14 @@ import { consoleManager } from '../../src/common/utils/console-manager'; import { getCommitDiff, extractFilesFromDiff } from '../utils/git-utils'; import { isDiagnosticLog } from '../utils/diagnostic-filter'; import { spawnSync } from 'child_process'; +import { + calculateWeightedAverage, + PillarName, + SEVEN_PILLARS, +} from '../../src/constants/agent-weights.constants'; +import { promptAndGenerateOkrs } from '../utils/okr-prompt.utils'; +import inquirer from 'inquirer'; +import pLimit from 'p-limit'; interface CommitInfo { hash: string; @@ -56,7 +65,7 @@ export async function runBatchEvaluateCommand(args: string[]) { // Load configuration const config = loadConfig(); if (!config) { - console.error('āŒ Config file not found. Run `npm run config` to create one.'); + console.error('āŒ Config file not found. Run `codewave config --init` to create one.'); process.exit(1); } validateConfig(config); @@ -89,7 +98,6 @@ export async function runBatchEvaluateCommand(args: string[]) { if (costEstimate !== null) { estimator.printEstimate(costEstimate, commits.length); - const { default: inquirer } = await import('inquirer'); const { proceed } = await inquirer.prompt([ { type: 'confirm', @@ -112,7 +120,6 @@ export async function runBatchEvaluateCommand(args: string[]) { const orchestrator = new CommitEvaluationOrchestrator(agentRegistry, config); // Configure concurrency limit (10 concurrent evaluations) - const { default: pLimit } = await import('p-limit'); const limit = pLimit(10); // Buffer for storing suppressed output (warnings, errors) @@ -248,8 +255,6 @@ export async function runBatchEvaluateCommand(args: string[]) { const authors = Array.from(uniqueAuthors); if (authors.length > 0) { - const { promptAndGenerateOkrs } = await import('../utils/okr-prompt.utils.js'); - const { getEvaluationRoot } = await import('../utils/shared.utils.js'); const evalRoot = getEvaluationRoot(); await promptAndGenerateOkrs(config, authors, evalRoot, { sinceDate: options.since ? new Date(options.since) : undefined, @@ -540,7 +545,7 @@ async function getCommitsToEvaluate(options: any): Promise { } function calculateAggregateMetrics(agentResults: any[]): any { - const metrics: any = { + const metrics: Record = { functionalImpact: 0, idealTimeHours: 0, testCoverage: 0, @@ -552,7 +557,6 @@ function calculateAggregateMetrics(agentResults: any[]): any { }; // Import weighted aggregation - const { calculateWeightedAverage } = require('../../src/constants/agent-weights.constants'); // Get latest metrics from each agent const agentMetricsMap = new Map(); @@ -563,8 +567,7 @@ function calculateAggregateMetrics(agentResults: any[]): any { }); // Calculate weighted average for each metric - const metricNames = Object.keys(metrics); - metricNames.forEach((metricName) => { + SEVEN_PILLARS.forEach((metricName: PillarName) => { const contributors: Array<{ agentName: string; score: number }> = []; agentMetricsMap.forEach((agentMetrics, agentRole) => { if (agentMetrics[metricName] !== undefined) { @@ -576,7 +579,10 @@ function calculateAggregateMetrics(agentResults: any[]): any { }); if (contributors.length > 0) { - metrics[metricName] = calculateWeightedAverage(contributors, metricName); + const weightedValue = calculateWeightedAverage(contributors, metricName); + if (weightedValue !== null) { + metrics[metricName] = weightedValue; + } } }); diff --git a/cli/commands/evaluate-command.ts b/cli/commands/evaluate-command.ts index a46b39d..21d2223 100644 --- a/cli/commands/evaluate-command.ts +++ b/cli/commands/evaluate-command.ts @@ -13,6 +13,7 @@ import { getEvaluationRoot, } from '../utils/shared.utils'; import { parseCommitStats } from '../../src/common/utils/commit-utils'; +import { promptAndGenerateOkrs } from '../utils/okr-prompt.utils'; import { getCommitDiff, getDiffFromStaged, @@ -384,7 +385,6 @@ export async function runEvaluateCommand(args: string[]) { printEvaluateCompletionMessage(outputDir); if (commitAuthor) { - const { promptAndGenerateOkrs } = await import('../utils/okr-prompt.utils.js'); const evalRoot = getEvaluationRoot(); await promptAndGenerateOkrs(config, [commitAuthor], evalRoot); } diff --git a/cli/commands/generate-okr-command.ts b/cli/commands/generate-okr-command.ts index 9190739..cdd6ab6 100644 --- a/cli/commands/generate-okr-command.ts +++ b/cli/commands/generate-okr-command.ts @@ -9,6 +9,7 @@ import { } from '../../src/services/author-stats-aggregator.service'; import { OkrOrchestrator } from '../../src/orchestrator/okr-orchestrator'; import { OkrProgressTracker } from '../utils/okr-progress-tracker'; +import { consoleManager } from '../../src/common/utils/console-manager'; /** * CLI command for generating OKRs @@ -62,7 +63,7 @@ export async function runGenerateOkrCommand(args: string[]) { console.log(chalk.gray(`āš” Using concurrency: ${concurrency}`)); // Suppress logs that interfere with progress bar - const { consoleManager } = await import('../../src/common/utils/console-manager.js'); + const originalStdoutWrite = process.stdout.write.bind(process.stdout); (process.stdout.write as any) = function (str: string, ...args: any[]): boolean { diff --git a/cli/index.ts b/cli/index.ts index f2297fe..4f9096b 100644 --- a/cli/index.ts +++ b/cli/index.ts @@ -21,6 +21,8 @@ import { runEvaluateCommand } from './commands/evaluate-command'; import { runConfigCommand } from './commands/config.command'; import { runBatchEvaluateCommand } from './commands/batch-evaluate-command'; import { runGenerateOkrCommand } from './commands/generate-okr-command'; +import * as path from 'path'; +import * as fs from 'fs'; async function main() { const [, , command, ...args] = process.argv; @@ -34,10 +36,9 @@ async function main() { if (command === '--version' || command === '-v') { try { // Try to load package.json from the project root - const path = require('path'); // __dirname is dist/cli, so go up 2 levels to reach root const packagePath = path.resolve(__dirname, '../../package.json'); - const packageJson = require(packagePath); + const packageJson = JSON.parse(fs.readFileSync(packagePath, 'utf-8')); console.log(`codewave version ${packageJson.version}`); } catch (error) { console.log('codewave version unknown'); diff --git a/cli/utils/okr-prompt.utils.ts b/cli/utils/okr-prompt.utils.ts index 9ee66d6..a4e7021 100644 --- a/cli/utils/okr-prompt.utils.ts +++ b/cli/utils/okr-prompt.utils.ts @@ -1,8 +1,17 @@ import chalk from 'chalk'; import { AppConfig } from '../../src/config/config.interface'; import { OkrOrchestrator } from '../../src/orchestrator/okr-orchestrator'; -import { AggregationOptions } from '../../src/services/author-stats-aggregator.service'; +import { + AggregationOptions, + AuthorStatsAggregatorService, +} from '../../src/services/author-stats-aggregator.service'; +import { MetricsCalculationService } from '../../src/services/metrics-calculation.service'; import { OkrProgressTracker } from './okr-progress-tracker'; +import inquirer from 'inquirer'; +import fs from 'fs'; +import path from 'path'; +import { generateAuthorPage } from './shared.utils'; +import { consoleManager } from '../../src/common/utils/console-manager'; /** * Prompt user and generate OKRs for authors @@ -31,7 +40,6 @@ export async function promptAndGenerateOkrs( console.log(chalk.gray(`Estimated cost: $${estimatedCost.toFixed(4)}`)); console.log(chalk.gray(`Authors: ${authors.join(', ')}`)); - const { default: inquirer } = await import('inquirer'); const { proceed } = await inquirer.prompt([ { type: 'confirm', @@ -59,7 +67,7 @@ export async function promptAndGenerateOkrs( tracker.initialize(authors); // Suppress logs that interfere with progress bar - const { consoleManager } = await import('../../src/common/utils/console-manager.js'); + const originalStdoutWrite = process.stdout.write.bind(process.stdout); (process.stdout.write as any) = function (str: string, ...args: any[]): boolean { @@ -103,12 +111,9 @@ export async function promptAndGenerateOkrs( /** * Regenerate author pages to include latest OKR data + * Uses consistent team summary calculation for BACI score accuracy */ async function regenerateAuthorPages(evalRoot: string, authors: string[]): Promise { - const fs = await import('fs'); - const path = await import('path'); - const { generateAuthorPage } = await import('./shared.utils.js'); - // Read index.json to get commit data for each author const indexPath = path.join(evalRoot, 'index.json'); if (!fs.existsSync(indexPath)) { @@ -128,11 +133,44 @@ async function regenerateAuthorPages(evalRoot: string, authors: string[]): Promi byAuthor.get(author)!.push(item); }); - // Regenerate pages only for authors with new OKRs + // Calculate team summary for BACI score consistency + const allAuthorData = await AuthorStatsAggregatorService.aggregateAuthorStats(evalRoot); + const allMetrics: any[] = []; + + // Convert all authors' evaluations to metrics format + for (const [authorName, authorEvaluations] of allAuthorData.entries()) { + const authorMetrics = authorEvaluations.map((evaluation) => { + // Calculate averaged metrics from agent results + const averagedMetrics = + evaluation.averagedMetrics || + MetricsCalculationService.calculateWeightedMetrics(evaluation.agents); + + return { + createdBy: authorName, + commitScore: averagedMetrics?.commitScore || 0, + testingQuality: averagedMetrics?.testCoverage || 0, + technicalDebtRate: 0, + deliveryRate: 0, + functionalImpact: averagedMetrics?.functionalImpact || 0, + codeQuality: averagedMetrics?.codeQuality || 0, + codeComplexity: averagedMetrics?.codeComplexity || 0, + actualTimeHours: averagedMetrics?.actualTimeHours || 0, + idealTimeHours: averagedMetrics?.idealTimeHours || 0, + technicalDebtHours: averagedMetrics?.technicalDebtHours || 0, + debtReductionHours: averagedMetrics?.debtReductionHours || 0, + }; + }); + allMetrics.push(...authorMetrics); + } + + // Get comprehensive team summary + const teamSummary = MetricsCalculationService.calculateTeamSummary(allMetrics); + + // Regenerate pages with consistent team summary for BACI score accuracy for (const author of authors) { const commits = byAuthor.get(author); if (commits && commits.length > 0) { - await generateAuthorPage(evalRoot, author, commits); + await generateAuthorPage(evalRoot, author, commits, teamSummary); console.log(chalk.gray(` āœ“ Updated dashboard for ${author}`)); } } diff --git a/cli/utils/shared.utils.ts b/cli/utils/shared.utils.ts index 3db2ca5..44088c4 100644 --- a/cli/utils/shared.utils.ts +++ b/cli/utils/shared.utils.ts @@ -10,15 +10,14 @@ import { DeveloperReviewerAgent } from '../../src/agents/implementations/develop import { AppConfig } from '../../src/config/config.interface'; import { generateEnhancedHtmlReport } from '../../src/formatters/html-report-formatter-enhanced'; import { generateConversationTranscript } from '../../src/formatters/conversation-transcript-formatter'; +import { MetricsCalculationService } from '../../src/services/metrics-calculation.service'; import { AgentResult } from '../../src/agents/agent.interface'; -import { - TokenSnapshot, - MetricsSnapshot, - EvaluationHistoryEntry, -} from '../../src/types/output.types'; +import { TokenSnapshot, EvaluationHistoryEntry } from '../../src/types/output.types'; import fs from 'fs/promises'; import * as fsSync from 'fs'; import path from 'path'; +import chalk from 'chalk'; +import { AuthorStatsAggregatorService } from '../../src/services/author-stats-aggregator.service'; /** * Generate timestamp in yyyyMMddHHmmss format @@ -183,56 +182,22 @@ export async function saveEvaluationReports(options: SaveReportsOptions): Promis /** * Extract metrics from final round agent results (last 5 agents) + * Extracts individual agent scores and applies weighted averaging based on agent expertise + * commitScore is calculated statically from the resulting weighted metrics */ -function extractMetricsSnapshot(agentResults: AgentResult[]): MetricsSnapshot { - // Get final round agents (last 5) - const finalAgents = agentResults.slice(-5); - - // Import weight functions for weighted averaging - const { - calculateWeightedAverage, - SEVEN_PILLARS, - } = require('../../src/constants/agent-weights.constants'); - - const metrics = SEVEN_PILLARS; - - const result: any = {}; - - // Calculate weighted average for each metric - metrics.forEach((metricName: string) => { - const contributors: Array<{ agentName: string; score: number | null }> = []; - finalAgents.forEach((agent) => { - if (agent.metrics && metricName in agent.metrics) { - const score = agent.metrics[metricName]; - contributors.push({ - agentName: agent.agentName || agent.agentRole || 'Unknown', - score: score !== null && score !== undefined ? score : null, - }); - } - }); - - if (contributors.length > 0) { - const weightedValue = calculateWeightedAverage(contributors, metricName); - // Determine decimal places based on metric - if (metricName.includes('Hours') || metricName.includes('Time')) { - result[metricName] = Number(weightedValue.toFixed(2)); - } else { - result[metricName] = Number(weightedValue.toFixed(1)); - } - } else { - result[metricName] = 0; - } - }); - +function extractMetricsSnapshot(agentResults: AgentResult[]): any { + // Use the centralized metrics calculation service for consistency + const metrics = MetricsCalculationService.calculateWeightedMetrics(agentResults); return { - functionalImpact: result.functionalImpact, - idealTimeHours: result.idealTimeHours, - testCoverage: result.testCoverage, - codeQuality: result.codeQuality, - codeComplexity: result.codeComplexity, - actualTimeHours: result.actualTimeHours, - technicalDebtHours: result.technicalDebtHours, - debtReductionHours: result.debtReductionHours, + functionalImpact: metrics.functionalImpact || 0, + idealTimeHours: metrics.idealTimeHours || 0, + testCoverage: metrics.testCoverage || 0, + codeQuality: metrics.codeQuality || 0, + codeComplexity: metrics.codeComplexity || 0, + actualTimeHours: metrics.actualTimeHours || 0, + technicalDebtHours: metrics.technicalDebtHours || 0, + debtReductionHours: metrics.debtReductionHours || 0, + commitScore: metrics.commitScore || 0, }; } @@ -329,6 +294,7 @@ async function trackEvaluationHistory( actualTimeHours: 0, technicalDebtHours: 0, debtReductionHours: 0, + commitScore: 0, }, tokens: agentResults ? extractTokenSnapshot(agentResults) @@ -433,9 +399,6 @@ export async function createEvaluationDirectory( * Calculate averaged metrics from agent results using weighted averaging (matching report calculations) */ async function calculateAveragedMetrics(evaluationDir: string): Promise { - const { MetricsCalculationService } = await import( - '../../src/services/metrics-calculation.service.js' - ); return MetricsCalculationService.loadMetricsFromDirectory(evaluationDir); } @@ -518,12 +481,6 @@ async function generateIndexHtml(indexPath: string, index: any[]): Promise byAuthor.get(author)!.push(item); }); - // Generate author pages for each author - const evaluationsRoot = path.dirname(indexPath); - for (const [author, commits] of byAuthor.entries()) { - await generateAuthorPage(evaluationsRoot, author, commits); - } - // Calculate overall metrics for display // Note: item.metrics already contains weighted consensus values from MetricsCalculationService // This is just aggregating those pre-calculated values for the summary stats @@ -533,6 +490,7 @@ async function generateIndexHtml(indexPath: string, index: any[]): Promise avgFunctionalImpact: 0, avgTestCoverage: 0, avgActualTime: 0, + avgCommitScore: 0, totalTechDebt: 0, count: 0, }; @@ -544,6 +502,7 @@ async function generateIndexHtml(indexPath: string, index: any[]): Promise overallMetrics.avgFunctionalImpact += item.metrics.functionalImpact || 0; overallMetrics.avgTestCoverage += item.metrics.testCoverage || 0; overallMetrics.avgActualTime += item.metrics.actualTimeHours || 0; + overallMetrics.avgCommitScore += item.metrics.commitScore || 0; // Calculate NET debt (debt introduced - debt reduction) const netDebt = (item.metrics.technicalDebtHours || 0) - (item.metrics.debtReductionHours || 0); @@ -568,9 +527,62 @@ async function generateIndexHtml(indexPath: string, index: any[]): Promise overallMetrics.avgActualTime = Number( (overallMetrics.avgActualTime / overallMetrics.count).toFixed(2) ); + overallMetrics.avgCommitScore = Number( + (overallMetrics.avgCommitScore / overallMetrics.count).toFixed(1) + ); overallMetrics.totalTechDebt = Number(overallMetrics.totalTechDebt.toFixed(2)); } + // Calculate team metrics using aggregated author data for consistency + const evalRoot = path.dirname(indexPath); + const allAuthorData = await AuthorStatsAggregatorService.aggregateAuthorStats(evalRoot); + const allMetrics: any[] = []; + + // Convert all authors' evaluations to metrics format + for (const [authorName, authorEvaluations] of allAuthorData.entries()) { + const authorMetrics = authorEvaluations.map((evaluation) => { + // Calculate averaged metrics from agent results + const averagedMetrics = + evaluation.averagedMetrics || + MetricsCalculationService.calculateWeightedMetrics(evaluation.agents); + + return { + createdBy: authorName, + commitScore: averagedMetrics?.commitScore || 0, + testingQuality: averagedMetrics?.testCoverage || 0, + technicalDebtRate: 0, + deliveryRate: 0, + functionalImpact: averagedMetrics?.functionalImpact || 0, + codeQuality: averagedMetrics?.codeQuality || 0, + codeComplexity: averagedMetrics?.codeComplexity || 0, + actualTimeHours: averagedMetrics?.actualTimeHours || 0, + idealTimeHours: averagedMetrics?.idealTimeHours || 0, + technicalDebtHours: averagedMetrics?.technicalDebtHours || 0, + debtReductionHours: averagedMetrics?.debtReductionHours || 0, + }; + }); + allMetrics.push(...authorMetrics); + } + + // Get comprehensive team summary (includes enhanced stats, BACI scores, and rankings) + const teamSummary = MetricsCalculationService.calculateTeamSummary(allMetrics); + + // Calculate average BACI score from team summary + const teamStats = Object.values(teamSummary.teamStats); + const avgBaciScore = + teamStats.length > 0 + ? Number( + ( + teamStats.reduce((sum, stat) => sum + (stat.baciScore || 0), 0) / teamStats.length + ).toFixed(1) + ) + : 0; + + // Generate author pages for each author (after team summary for BACI score consistency) + for (const [author, commits] of byAuthor.entries()) { + await generateAuthorPage(evalRoot, author, commits, teamSummary); + } + const html = ` @@ -624,10 +636,23 @@ async function generateIndexHtml(indexPath: string, index: any[]): Promise #commitsTable th:nth-child(8) { min-width: 95px; } /* Complexity */ #commitsTable th:nth-child(9) { min-width: 80px; } /* Tests */ #commitsTable th:nth-child(10) { min-width: 85px; } /* Impact */ - #commitsTable th:nth-child(11) { min-width: 80px; } /* Time */ - #commitsTable th:nth-child(12) { min-width: 90px; } /* Tech Debt */ - #commitsTable th:nth-child(13) { min-width: 75px; } /* Action */ + #commitsTable th:nth-child(11) { min-width: 95px; } /* Commit Score */ + #commitsTable th:nth-child(12) { min-width: 80px; } /* Time */ + #commitsTable th:nth-child(13) { min-width: 90px; } /* Tech Debt */ + #commitsTable th:nth-child(14) { min-width: 75px; } /* Action */ #commitsTable td:nth-child(3) { max-width: 280px; word-wrap: break-word; overflow-wrap: break-word; } /* Message text wrapping */ + /* Authors table column sizing */ + .table-authors th:nth-child(1) { min-width: 120px; } /* Author */ + .table-authors th:nth-child(2) { min-width: 80px; } /* Commits */ + .table-authors th:nth-child(3) { min-width: 90px; } /* Score */ + .table-authors th:nth-child(4) { min-width: 90px; } /* Avg Quality */ + .table-authors th:nth-child(5) { min-width: 95px; } /* Avg Complexity */ + .table-authors th:nth-child(6) { min-width: 80px; } /* Avg Tests */ + .table-authors th:nth-child(7) { min-width: 85px; } /* Avg Impact */ + .table-authors th:nth-child(8) { min-width: 95px; } /* Avg Commit Score */ + .table-authors th:nth-child(9) { min-width: 80px; } /* Avg Time */ + .table-authors th:nth-child(10) { min-width: 90px; } /* Total Tech Debt */ + .table-authors th:nth-child(11) { min-width: 110px; } /* Action */ @@ -657,6 +682,11 @@ async function generateIndexHtml(indexPath: string, index: any[]): Promise ${ overallMetrics.count > 0 ? ` +
+
${avgBaciScore}
+
Avg Score
+
out of 10
+
${overallMetrics.avgQuality}
Avg Quality
@@ -677,6 +707,11 @@ async function generateIndexHtml(indexPath: string, index: any[]): Promise
Avg Impact
out of 10
+
+
${overallMetrics.avgCommitScore}
+
Avg Commit Score
+
out of 10
+
${overallMetrics.avgActualTime}h
Avg Time
@@ -707,10 +742,12 @@ async function generateIndexHtml(indexPath: string, index: any[]): Promise Author Commits + Score Avg Quality Avg Complexity Avg Tests Avg Impact + Avg Commit Score Avg Time Total Tech Debt Action @@ -718,8 +755,26 @@ async function generateIndexHtml(indexPath: string, index: any[]): Promise ${Array.from(byAuthor.entries()) - .sort((a, b) => b[1].length - a[1].length) .map(([author, commits]) => { + // Calculate average commit score once + let totalScore = 0; + let count = 0; + commits.forEach((c) => { + if (c.metrics && typeof c.metrics.commitScore === 'number') { + totalScore += c.metrics.commitScore; + count++; + } + }); + const avgCommitScore = count > 0 ? totalScore / count : 0; + + return { author, commits, avgCommitScore }; + }) + .sort( + (a, b) => + (teamSummary.teamStats[b.author]?.baciScore || 0) - + (teamSummary.teamStats[a.author]?.baciScore || 0) + ) // Sort by BACI score + .map(({ author, commits }) => { // Sort commits by commit date (newest first) commits.sort((a, b) => new Date(b.commitDate).getTime() - new Date(a.commitDate).getTime()); const authorMetrics = { @@ -727,6 +782,7 @@ ${Array.from(byAuthor.entries()) complexity: 0, testCoverage: 0, functionalImpact: 0, + commitScore: 0, actualTime: 0, techDebt: 0, count: 0, @@ -737,6 +793,7 @@ ${Array.from(byAuthor.entries()) authorMetrics.complexity += c.metrics.codeComplexity || 0; authorMetrics.testCoverage += c.metrics.testCoverage || 0; authorMetrics.functionalImpact += c.metrics.functionalImpact || 0; + authorMetrics.commitScore += c.metrics.commitScore || 0; authorMetrics.actualTime += c.metrics.actualTimeHours || 0; // Calculate NET debt (debt introduced - debt reduction) const netDebt = (c.metrics.technicalDebtHours || 0) - (c.metrics.debtReductionHours || 0); @@ -757,6 +814,10 @@ ${Array.from(byAuthor.entries()) authorMetrics.count > 0 ? (authorMetrics.functionalImpact / authorMetrics.count).toFixed(1) : 'N/A'; + const displayAvgCommitScore = + authorMetrics.count > 0 + ? (authorMetrics.commitScore / authorMetrics.count).toFixed(1) + : 'N/A'; const avgActualTime = authorMetrics.count > 0 ? (authorMetrics.actualTime / authorMetrics.count).toFixed(2) : 'N/A'; const totalTechDebt = authorMetrics.count > 0 ? authorMetrics.techDebt.toFixed(2) : 'N/A'; @@ -768,10 +829,17 @@ ${Array.from(byAuthor.entries()) šŸ‘¤ ${author} ${commits.length} + ${(() => { + const baciScore = teamSummary.teamStats[author]?.baciScore || 0; + return baciScore > 0 + ? `${baciScore.toFixed(1)}/10` + : 'N/A'; + })()} ${avgQuality !== 'N/A' ? `${avgQuality}/10` : 'N/A'} ${avgComplexity !== 'N/A' ? `${avgComplexity}/10` : 'N/A'} ${avgTestCoverage !== 'N/A' ? `${avgTestCoverage}/10` : 'N/A'} - ${avgFunctionalImpact !== 'N/A' ? `${avgFunctionalImpact}/10` : 'N/A'} + ${avgFunctionalImpact !== 'N/A' ? `${avgFunctionalImpact}/10` : 'N/A'} + ${displayAvgCommitScore !== 'N/A' ? `${displayAvgCommitScore}/10` : 'N/A'} ${avgActualTime !== 'N/A' ? `${avgActualTime}h` : 'N/A'} ${totalTechDebt !== 'N/A' ? `${parseFloat(totalTechDebt) > 0 ? '+' : ''}${totalTechDebt}h` : 'N/A'} View Dashboard @@ -798,6 +866,7 @@ ${Array.from(byAuthor.entries()) Complexity Tests Impact + Commit Score Time Tech Debt Action @@ -814,7 +883,9 @@ ${index const testsColor = metrics.testCoverage >= 7 ? 'good' : metrics.testCoverage >= 4 ? 'medium' : 'bad'; const impactColor = - metrics.functionalImpact >= 7 ? 'bad' : metrics.functionalImpact >= 4 ? 'medium' : 'good'; + metrics.functionalImpact >= 7 ? 'good' : metrics.functionalImpact >= 4 ? 'medium' : 'bad'; + const commitScoreColor = + metrics.commitScore >= 7 ? 'good' : metrics.commitScore >= 4 ? 'medium' : 'bad'; const netDebt = (metrics.technicalDebtHours || 0) - (metrics.debtReductionHours || 0); const debtColor = netDebt > 0 ? 'bad' : netDebt < 0 ? 'good' : 'medium'; @@ -835,6 +906,7 @@ ${index ${item.metrics ? `${metrics.codeComplexity}/10` : 'N/A'} ${item.metrics ? `${metrics.testCoverage}/10` : 'N/A'} ${item.metrics ? `${metrics.functionalImpact}/10` : 'N/A'} + ${item.metrics && typeof metrics.commitScore === 'number' ? `${metrics.commitScore}/10` : 'N/A'} ${item.metrics ? `${metrics.actualTimeHours}h` : 'N/A'} ${item.metrics ? `${netDebt > 0 ? '+' : ''}${netDebt.toFixed(1)}h` : 'N/A'} View @@ -930,15 +1002,13 @@ export function generateBatchIdentifier(options: { export async function generateAuthorPage( evaluationsRoot: string, author: string, - commits: any[] + commits: any[], + teamSummary: any ): Promise { const authorSlug = author.toLowerCase().replace(/[^a-z0-9]/g, '_'); const authorPagePath = path.join(evaluationsRoot, `author-${authorSlug}.html`); // Use centralized metrics calculation service for consistency with OKR generation - const { - AuthorStatsAggregatorService, - } = require('../../src/services/author-stats-aggregator.service'); const authorData = await AuthorStatsAggregatorService.aggregateAuthorStats(evaluationsRoot, { targetAuthor: author, }); @@ -964,8 +1034,12 @@ export async function generateAuthorPage( const avgFunctionalImpact = analysis ? analysis.stats.impact.toFixed(1) : 'N/A'; const avgActualTime = analysis ? analysis.stats.time.toFixed(2) : 'N/A'; const totalTechDebt = analysis ? analysis.stats.techDebt.toFixed(2) : 'N/A'; + const avgCommitScore = analysis ? analysis.stats.commitScore.toFixed(1) : 'N/A'; const commitCount = deduplicatedCommits.length; // Use deduplicated count + // Get BACI score from the provided team summary + const avgBaciScore = teamSummary?.teamStats?.[author]?.baciScore?.toFixed(1) || 'N/A'; + // Check for OKR files and load latest OKR data const okrsDir = path.join(evaluationsRoot, '.okrs', authorSlug); let okrContentHtml = ''; @@ -1322,6 +1396,11 @@ export async function generateAuthorPage(
${commits.length}
Total Commits
+
+
${avgBaciScore}
+
Avg Score
+
out of 10
+
${avgQuality}
Avg Quality
@@ -1342,6 +1421,11 @@ export async function generateAuthorPage(
Avg Impact
out of 10
+
+
${avgCommitScore}
+
Avg Commit Score
+
out of 10
+
${avgActualTime}h
Avg Time
@@ -1377,6 +1461,7 @@ export async function generateAuthorPage( Complexity Tests Impact + Commit Score Time Tech Debt Action @@ -1386,6 +1471,9 @@ export async function generateAuthorPage( ${commits .map((item) => { const metrics = item.metrics || {}; + // Calculate NET debt (debt introduced - debt reduction) + const netDebt = (metrics.technicalDebtHours || 0) - (metrics.debtReductionHours || 0); + const qualityColor = metrics.codeQuality >= 7 ? 'good' : metrics.codeQuality >= 4 ? 'medium' : 'bad'; const complexityColor = @@ -1393,9 +1481,11 @@ ${commits const testsColor = metrics.testCoverage >= 7 ? 'good' : metrics.testCoverage >= 4 ? 'medium' : 'bad'; const impactColor = - metrics.functionalImpact >= 7 ? 'bad' : metrics.functionalImpact >= 4 ? 'medium' : 'good'; + metrics.functionalImpact >= 7 ? 'good' : metrics.functionalImpact >= 4 ? 'medium' : 'bad'; + const commitScoreColor = + metrics.commitScore >= 7 ? 'good' : metrics.commitScore >= 4 ? 'medium' : 'bad'; const debtColor = - metrics.technicalDebtHours > 0 ? 'bad' : metrics.technicalDebtHours < 0 ? 'good' : 'medium'; + netDebt > 0 ? 'bad' : netDebt < 0 ? 'good' : 'medium'; return ` @@ -1413,8 +1503,9 @@ ${commits ${item.metrics ? `${metrics.codeComplexity}/10` : 'N/A'} ${item.metrics ? `${metrics.testCoverage}/10` : 'N/A'} ${item.metrics ? `${metrics.functionalImpact}/10` : 'N/A'} + ${item.metrics ? `${metrics.commitScore}/10` : 'N/A'} ${item.metrics ? `${metrics.actualTimeHours}h` : 'N/A'} - ${item.metrics ? `${metrics.technicalDebtHours > 0 ? '+' : ''}${metrics.technicalDebtHours}h` : 'N/A'} + ${item.metrics ? `${netDebt > 0 ? '+' : ''}${netDebt.toFixed(1)}h` : 'N/A'} View `; }) @@ -1466,8 +1557,6 @@ export function buildIndexUrl(): string { */ export function printEvaluateCompletionMessage(outputDir: string): void { try { - const chalk = require('chalk').default; - console.log(chalk.green(`\nāœ… Evaluation complete!`)); console.log(chalk.cyan(`šŸ“ Output directory: ${chalk.bold(outputDir)}`)); console.log( diff --git a/package-lock.json b/package-lock.json index a5442ed..87cd80d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ }, "devDependencies": { "@types/glob": "^8.1.0", + "@types/lodash": "^4.17.21", "@types/node": "^20.19.24", "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", @@ -324,6 +325,7 @@ "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.1.tgz", "integrity": "sha512-vdUoj2CVbb+0Qszi8llP34vdUCfP7bfA9VoFr4Se1pFGu7VAPnk8lBnRat9IvqSxMfTvOHJSd7Rn6TUPjzKsnA==", "license": "MIT", + "peer": true, "dependencies": { "@cfworker/json-schema": "^4.0.2", "ansi-styles": "^5.0.0", @@ -606,6 +608,13 @@ "rxjs": "^7.2.0" } }, + "node_modules/@types/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-FOvQ0YPD5NOfPgMzJihoT+Za5pdkDJWcbpuj1DjaKZIr/gxodQjY/uWEFlTNqW2ugXHUiL8lRQgw63dzKHZdeQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/minimatch": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", @@ -693,6 +702,7 @@ "integrity": "sha512-4Z+L8I2OqhZV8qA132M4wNL30ypZGYOQVBfMgxDH/K5UX0PNqTu1c6za9ST5r9+tavvHiTWmBnKzpCJ/GlVFtg==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "7.18.0", "@typescript-eslint/types": "7.18.0", @@ -871,6 +881,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1455,6 +1466,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -1511,6 +1523,7 @@ "integrity": "sha512-iI1f+D2ViGn+uvv5HuHVUamg8ll4tN+JRHGc6IJi4TP9Kl976C57fzPXgseXNs8v0iA8aSJpHsTWjDb9QJamGQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -3024,6 +3037,7 @@ "integrity": "sha512-QgODejq9K3OzoBbuyobZlUhznP5SKwPqp+6Q6xw6o8gnhr4O85L2U915iM2IDcfF2NPXVaM9zlo9tdwipnYwzg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -3530,6 +3544,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -3726,6 +3741,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.13.tgz", "integrity": "sha512-AvvthqfqrAhNH9dnfmrfKzX5upOdjUVJYFqNSlkmGf64gRaTzlPwz99IHYnVs28qYAybvAlBV+H7pn0saFY4Ig==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 59c2d92..40aafd3 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "type": "commonjs", "devDependencies": { "@types/glob": "^8.1.0", + "@types/lodash": "^4.17.21", "@types/node": "^20.19.24", "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^7.0.0", diff --git a/src/agents/core/base-agent.ts b/src/agents/core/base-agent.ts index 689739e..95d25e4 100644 --- a/src/agents/core/base-agent.ts +++ b/src/agents/core/base-agent.ts @@ -14,6 +14,7 @@ import { Agent, AgentContext, AgentResult, AgentExecutionOptions } from '../agent.interface'; import { AgentMetadata, AgentExpertise, categorizeExpertise } from './agent-metadata'; +import { AgentExecutor } from '../execution/agent-executor'; import { PromptContext } from '../prompts/prompt-builder.interface'; import { AGENT_METRIC_DEFINITIONS, @@ -115,9 +116,6 @@ export abstract class BaseAgent implements Agent { * This delegates to AgentExecutor for actual execution */ async execute(context: AgentContext, options?: AgentExecutionOptions): Promise { - // Lazy-load the executor to avoid circular dependencies - const { AgentExecutor } = await import('../execution/agent-executor.js'); - const executor = new AgentExecutor(this.config, this.metadata, this.systemInstructions); // Build prompt context with categorized expertise diff --git a/src/agents/execution/agent-internal-graph.ts b/src/agents/execution/agent-internal-graph.ts index bb26f2e..a5a882c 100644 --- a/src/agents/execution/agent-internal-graph.ts +++ b/src/agents/execution/agent-internal-graph.ts @@ -43,7 +43,7 @@ export const AgentInternalState = Annotation.Root({ // Messages for LLM conversation messages: Annotation({ - reducer: (state: BaseMessage[], update: BaseMessage[]) => [...state, ...update], + reducer: (state: BaseMessage[], update: BaseMessage[]) => (update.length > 0 ? update : state), default: () => [], }), diff --git a/src/agents/implementations/business-analyst-agent.ts b/src/agents/implementations/business-analyst-agent.ts index 5752565..193c5ac 100644 --- a/src/agents/implementations/business-analyst-agent.ts +++ b/src/agents/implementations/business-analyst-agent.ts @@ -8,6 +8,8 @@ import { BaseAgent } from '../core/base-agent'; import { AgentMetadata, AgentExpertise } from '../core/agent-metadata'; import { PromptContext } from '../prompts/prompt-builder.interface'; +import { CombinedRAGHelper } from '../../utils/combined-rag-helper'; +import { getInitialQueriesForRole } from '../../utils/gap-to-rag-query-mapper'; export class BusinessAnalystAgent extends BaseAgent { // ============================================================================ @@ -64,9 +66,6 @@ Return your analysis as JSON with all 8 metrics, even if some are outside your p // Use RAG if available let contentSection = ''; if (context.vectorStore || context.documentationStore) { - const { CombinedRAGHelper } = await import('../../utils/combined-rag-helper.js'); - const { getInitialQueriesForRole } = await import('../../utils/gap-to-rag-query-mapper.js'); - const rag = new CombinedRAGHelper(context.vectorStore, context.documentationStore); rag.setAgentName(this.metadata.role); diff --git a/src/agents/implementations/developer-author-agent.ts b/src/agents/implementations/developer-author-agent.ts index 9a0690a..0564697 100644 --- a/src/agents/implementations/developer-author-agent.ts +++ b/src/agents/implementations/developer-author-agent.ts @@ -8,7 +8,8 @@ import { BaseAgent } from '../core/base-agent'; import { AgentMetadata, AgentExpertise } from '../core/agent-metadata'; import { PromptContext } from '../prompts/prompt-builder.interface'; - +import { CombinedRAGHelper } from '../../utils/combined-rag-helper'; +import { getInitialQueriesForRole } from '../../utils/gap-to-rag-query-mapper'; export class DeveloperAuthorAgent extends BaseAgent { // ============================================================================ // AGENT IDENTITY & EXPERTISE @@ -67,8 +68,6 @@ Return your analysis as JSON with all 8 metrics, even if some are outside your p // Use RAG if available let contentSection = ''; if (context.vectorStore || context.documentationStore) { - const { CombinedRAGHelper } = await import('../../utils/combined-rag-helper.js'); - const { getInitialQueriesForRole } = await import('../../utils/gap-to-rag-query-mapper.js'); const rag = new CombinedRAGHelper(context.vectorStore, context.documentationStore); rag.setAgentName(this.metadata.role); diff --git a/src/agents/implementations/developer-reviewer-agent.ts b/src/agents/implementations/developer-reviewer-agent.ts index 8327d1e..689b375 100644 --- a/src/agents/implementations/developer-reviewer-agent.ts +++ b/src/agents/implementations/developer-reviewer-agent.ts @@ -8,7 +8,8 @@ import { BaseAgent } from '../core/base-agent'; import { AgentMetadata, AgentExpertise } from '../core/agent-metadata'; import { PromptContext } from '../prompts/prompt-builder.interface'; - +import { CombinedRAGHelper } from '../../utils/combined-rag-helper'; +import { getInitialQueriesForRole } from '../../utils/gap-to-rag-query-mapper'; export class DeveloperReviewerAgent extends BaseAgent { // ============================================================================ // AGENT IDENTITY & EXPERTISE @@ -68,8 +69,6 @@ Return your analysis as JSON with all 8 metrics, even if some are outside your p // Use RAG if available let contentSection = ''; if (context.vectorStore || context.documentationStore) { - const { CombinedRAGHelper } = await import('../../utils/combined-rag-helper.js'); - const { getInitialQueriesForRole } = await import('../../utils/gap-to-rag-query-mapper.js'); const rag = new CombinedRAGHelper(context.vectorStore, context.documentationStore); rag.setAgentName(this.metadata.role); diff --git a/src/agents/implementations/sdet-agent.ts b/src/agents/implementations/sdet-agent.ts index 520003d..66fc476 100644 --- a/src/agents/implementations/sdet-agent.ts +++ b/src/agents/implementations/sdet-agent.ts @@ -8,6 +8,8 @@ import { BaseAgent } from '../core/base-agent'; import { AgentMetadata, AgentExpertise } from '../core/agent-metadata'; import { PromptContext } from '../prompts/prompt-builder.interface'; +import { CombinedRAGHelper } from '../../utils/combined-rag-helper'; +import { getInitialQueriesForRole } from '../../utils/gap-to-rag-query-mapper'; export class SDETAgent extends BaseAgent { // ============================================================================ @@ -65,9 +67,6 @@ Return your analysis as JSON with all 8 metrics, even if some are outside your p // Use RAG if available let contentSection = ''; if (context.vectorStore || context.documentationStore) { - const { CombinedRAGHelper } = await import('../../utils/combined-rag-helper.js'); - const { getInitialQueriesForRole } = await import('../../utils/gap-to-rag-query-mapper.js'); - const rag = new CombinedRAGHelper(context.vectorStore, context.documentationStore); rag.setAgentName(this.metadata.role); diff --git a/src/agents/implementations/senior-architect-agent.ts b/src/agents/implementations/senior-architect-agent.ts index a1ca2c2..2d3fc2a 100644 --- a/src/agents/implementations/senior-architect-agent.ts +++ b/src/agents/implementations/senior-architect-agent.ts @@ -8,7 +8,8 @@ import { BaseAgent } from '../core/base-agent'; import { AgentMetadata, AgentExpertise } from '../core/agent-metadata'; import { PromptContext } from '../prompts/prompt-builder.interface'; - +import { CombinedRAGHelper } from '../../utils/combined-rag-helper'; +import { getInitialQueriesForRole } from '../../utils/gap-to-rag-query-mapper'; export class SeniorArchitectAgent extends BaseAgent { // ============================================================================ // AGENT IDENTITY & EXPERTISE @@ -68,8 +69,6 @@ Return your analysis as JSON with all 7 metrics, even if some are outside your p // Use RAG if available let contentSection = ''; if (context.vectorStore || context.documentationStore) { - const { CombinedRAGHelper } = await import('../../utils/combined-rag-helper.js'); - const { getInitialQueriesForRole } = await import('../../utils/gap-to-rag-query-mapper.js'); const rag = new CombinedRAGHelper(context.vectorStore, context.documentationStore); rag.setAgentName(this.metadata.role); diff --git a/src/constants/agent-weights.constants.ts b/src/constants/agent-weights.constants.ts index 9f707db..7801089 100644 --- a/src/constants/agent-weights.constants.ts +++ b/src/constants/agent-weights.constants.ts @@ -164,7 +164,7 @@ export function normalizeAgentName(agentName: string): string { } // Helper: Get agent's weight for a specific pillar -export function getAgentWeight(agentName: string, pillar: keyof AgentWeights): number { +export function getAgentWeight(agentName: string, pillar: PillarName): number { // Normalize agent name before lookup const normalizedName = normalizeAgentName(agentName); const weights = AGENT_EXPERTISE_WEIGHTS[normalizedName]; @@ -177,7 +177,7 @@ export function getAgentWeight(agentName: string, pillar: keyof AgentWeights): n } // Helper: Get all agents' weights for a specific pillar -export function getPillarWeights(pillar: keyof AgentWeights): Record { +export function getPillarWeights(pillar: PillarName): Record { const result: Record = {}; for (const [agentName, weights] of Object.entries(AGENT_EXPERTISE_WEIGHTS)) { result[agentName] = weights[pillar]; @@ -188,7 +188,7 @@ export function getPillarWeights(pillar: keyof AgentWeights): Record, - pillar: keyof AgentWeights + pillar: PillarName ): number | null { // Filter out null values before calculating average const validScores = scores.filter( diff --git a/src/formatters/html-report-formatter-enhanced.ts b/src/formatters/html-report-formatter-enhanced.ts index 0a08f6c..7029e68 100644 --- a/src/formatters/html-report-formatter-enhanced.ts +++ b/src/formatters/html-report-formatter-enhanced.ts @@ -4,6 +4,12 @@ import fs from 'fs'; import path from 'path'; import { AgentResult } from '../agents/agent.interface'; import { EvaluationHistoryEntry } from '../types/output.types'; +import { + SEVEN_PILLARS, + getAgentWeight, + calculateWeightedAverage, + PillarName, +} from '../constants/agent-weights.constants'; interface AgentEvaluation { agentName: string; @@ -306,13 +312,6 @@ function groupResultsByAgent(results: AgentResult[]): Map ): MetricEvolution[] { - // Import centralized pillar constants and weight functions - const { - SEVEN_PILLARS, - getAgentWeight, - calculateWeightedAverage, - } = require('../constants/agent-weights.constants'); - const metricMap = new Map(); // Group all evaluations by round @@ -327,7 +326,7 @@ function calculateMetricEvolution( }); // For each metric, calculate consensus score per round - SEVEN_PILLARS.forEach((metric: string) => { + SEVEN_PILLARS.forEach((metric: PillarName) => { const metricEvolution: MetricEvolution = { metric, rounds: new Map(), @@ -356,7 +355,7 @@ function calculateMetricEvolution( const consensusScore = calculateWeightedAverage( contributors.map((c) => ({ agentName: c.agentName, score: c.score })), metric - ); + ) || 0; metricEvolution.rounds.set(round, consensusScore); // Check if value changed from first round @@ -385,19 +384,13 @@ function calculateConsensusValues(groupedResults: Map contributors: Array<{ name: string; score: number | null; weight: number }>; } > { - const { - getAgentWeight, - calculateWeightedAverage, - SEVEN_PILLARS, - } = require('../constants/agent-weights.constants'); - // Collect metrics const allMetrics = new Set(); groupedResults.forEach((evaluations) => { evaluations.forEach((evaluation) => { if (evaluation.metrics) { Object.keys(evaluation.metrics) - .filter((metric) => SEVEN_PILLARS.includes(metric)) + .filter((metric) => SEVEN_PILLARS.includes(metric as PillarName)) .forEach((metric) => allMetrics.add(metric)); } }); @@ -412,7 +405,8 @@ function calculateConsensusValues(groupedResults: Map const filteredMetrics = Object.fromEntries( Object.entries(latestEval.metrics).filter( ([metric, value]) => - SEVEN_PILLARS.includes(metric) && (typeof value === 'number' || value === null) + SEVEN_PILLARS.includes(metric as PillarName) && + (typeof value === 'number' || value === null) ) ); agentMetrics.set(agentName, new Map(Object.entries(filteredMetrics))); @@ -436,7 +430,7 @@ function calculateConsensusValues(groupedResults: Map if (metrics.has(metric)) { const score = metrics.get(metric)!; // Can be number or null const agentKey = agentRoleMap.get(agentName) || agentName; - const weight = getAgentWeight(agentKey, metric); + const weight = getAgentWeight(agentKey, metric as PillarName); contributors.push({ name: agentName, score, weight }); } }); @@ -446,7 +440,7 @@ function calculateConsensusValues(groupedResults: Map agentName: agentRoleMap.get(c.name) || c.name, score: c.score, })), - metric + metric as PillarName ); finalValues.set(metric, { value: weightedAvg, contributors }); } @@ -459,20 +453,13 @@ function calculateConsensusValues(groupedResults: Map * Build comprehensive metrics table showing all agent contributions */ function buildMetricsTable(groupedResults: Map): string { - // Import agent weights and centralized pillar constants - const { - getAgentWeight, - calculateWeightedAverage, - SEVEN_PILLARS, - } = require('../constants/agent-weights.constants'); - // Collect metrics, filtering to ONLY the 7 pillars const allMetrics = new Set(); groupedResults.forEach((evaluations) => { evaluations.forEach((evaluation) => { if (evaluation.metrics) { Object.keys(evaluation.metrics) - .filter((metric) => SEVEN_PILLARS.includes(metric)) + .filter((metric) => SEVEN_PILLARS.includes(metric as PillarName)) .forEach((metric) => allMetrics.add(metric)); } }); @@ -488,7 +475,8 @@ function buildMetricsTable(groupedResults: Map): stri const filteredMetrics = Object.fromEntries( Object.entries(latestEval.metrics).filter( ([metric, value]) => - SEVEN_PILLARS.includes(metric) && (typeof value === 'number' || value === null) + SEVEN_PILLARS.includes(metric as PillarName) && + (typeof value === 'number' || value === null) ) ); agentMetrics.set(agentName, new Map(Object.entries(filteredMetrics))); @@ -514,7 +502,7 @@ function buildMetricsTable(groupedResults: Map): stri const score = metrics.get(metric)!; // Can be number or null // Use agentRole (technical key) for weight lookup, fallback to agentName const agentKey = agentRoleMap.get(agentName) || agentName; - const weight = getAgentWeight(agentKey, metric); + const weight = getAgentWeight(agentKey, metric as PillarName); contributors.push({ name: agentName, score, weight }); } }); @@ -525,7 +513,7 @@ function buildMetricsTable(groupedResults: Map): stri agentName: agentRoleMap.get(c.name) || c.name, // Use agentRole for weight lookup score: c.score, })), - metric + metric as PillarName ); finalValues.set(metric, { value: weightedAvg, contributors }); } @@ -567,7 +555,7 @@ function buildMetricsTable(groupedResults: Map): stri const value = agentMetrics.get(agent)?.get(metric); // Use agentRole for weight lookup const agentKey = agentRoleMap.get(agent) || agent; - const weight = getAgentWeight(agentKey, metric); + const weight = getAgentWeight(agentKey, metric as PillarName); // Ensure weight is a number before calling toFixed if (typeof weight !== 'number') { @@ -705,9 +693,8 @@ function generateHistoryHtml(history: EvaluationHistoryEntry[], modelInfo?: stri } // Build comparison tables - Evaluations as ROWS, Metrics as COLUMNS - const { SEVEN_PILLARS } = require('../constants/agent-weights.constants'); const allMetrics = SEVEN_PILLARS; - const stats = calculateHistoryStatistics(history, allMetrics); + const stats = calculateHistoryStatistics(history, [...allMetrics]); // Build evaluation rows (each row is one evaluation with all metrics as columns) const evaluationRows = history diff --git a/src/formatters/html-report-formatter.ts b/src/formatters/html-report-formatter.ts index 545c6c0..5c9a224 100644 --- a/src/formatters/html-report-formatter.ts +++ b/src/formatters/html-report-formatter.ts @@ -151,7 +151,8 @@ export function generateHtmlReport( if ( key.toLowerCase().includes('quality') || key.toLowerCase().includes('coverage') || - key.toLowerCase().includes('impact') + key.toLowerCase().includes('impact') || + key.toLowerCase().includes('commitscore') ) { badge = value >= 7 ? 'success' : value >= 4 ? 'warning' : 'danger'; } diff --git a/src/orchestrator/commit-evaluation-graph.ts b/src/orchestrator/commit-evaluation-graph.ts index 9b96c55..f392358 100644 --- a/src/orchestrator/commit-evaluation-graph.ts +++ b/src/orchestrator/commit-evaluation-graph.ts @@ -4,7 +4,14 @@ import { AgentResult } from '../agents/agent.interface'; import { ConversationMessage, PillarScores } from '../types/agent.types'; import { AppConfig } from '../config/config.interface'; import { calculateCost } from '../utils/token-tracker'; -import { SEVEN_PILLARS, PillarName } from '../constants/agent-weights.constants'; +import { + SEVEN_PILLARS, + PillarName, + calculateWeightedAverage, + getAgentWeight, +} from '../constants/agent-weights.constants'; +import { DeveloperOverviewGenerator } from '../services/developer-overview-generator'; +import { LLMService } from '../llm/llm-service'; /** * LangGraph State Definition for Commit Evaluation @@ -276,10 +283,6 @@ export function createCommitEvaluationGraph(agentRegistry: AgentRegistry, config console.log('šŸ“ Generating developer overview from commit diff...'); - const { DeveloperOverviewGenerator } = await import( - '../services/developer-overview-generator.js' - ); - const { LLMService } = await import('../llm/llm-service.js'); const generator = new DeveloperOverviewGenerator(config); const overview = await generator.generateOverview( @@ -501,11 +504,6 @@ export function createCommitEvaluationGraph(agentRegistry: AgentRegistry, config ); } - // Import centralized constants and utilities - const { - SEVEN_PILLARS, - calculateWeightedAverage, - } = require('../constants/agent-weights.constants'); // Sanitize results: filter metrics to ONLY the 7 pillars const results = validResponses.map((r) => { @@ -557,7 +555,6 @@ export function createCommitEvaluationGraph(agentRegistry: AgentRegistry, config } // Validate: Warn if agents return null for their PRIMARY metrics (weight >= 0.4) - const { getAgentWeight } = require('../constants/agent-weights.constants'); for (const result of results) { const agentName = result.agentRole || result.agentName || 'unknown'; if (result.metrics) { diff --git a/src/orchestrator/okr-orchestrator.ts b/src/orchestrator/okr-orchestrator.ts index a64b211..72da174 100644 --- a/src/orchestrator/okr-orchestrator.ts +++ b/src/orchestrator/okr-orchestrator.ts @@ -9,6 +9,7 @@ import { import { OkrAgentService } from '../services/okr-agent.service'; import { formatOKRToMarkdown } from '../formatters/okr-formatter'; import { formatOKRToHTML } from '../formatters/okr-html-formatter'; +import pLimit from 'p-limit'; /** * Progress information for a single author's OKR generation @@ -65,7 +66,6 @@ export class OkrOrchestrator { silent: boolean = false, onProgress?: (author: string, progress: OkrProgress) => void ): Promise> { - const pLimit = (await import('p-limit')).default; const limit = pLimit(concurrency); // Initialize progress tracking diff --git a/src/services/author-stats-aggregator.service.ts b/src/services/author-stats-aggregator.service.ts index 31d7299..6383856 100644 --- a/src/services/author-stats-aggregator.service.ts +++ b/src/services/author-stats-aggregator.service.ts @@ -3,6 +3,7 @@ import * as fs from 'fs'; import * as path from 'path'; +import { MetricsCalculationService } from './metrics-calculation.service'; export interface AuthorStats { commits: number; @@ -12,6 +13,7 @@ export interface AuthorStats { impact: number; time: number; techDebt: number; + commitScore: number; } export interface AggregationOptions { @@ -139,7 +141,6 @@ export class AuthorStatsAggregatorService { * Delegates to centralized MetricsCalculationService */ static calculateAverageMetrics(evaluations: any[]): AuthorStats { - const { MetricsCalculationService } = require('./metrics-calculation.service'); return MetricsCalculationService.calculateSimpleAverageMetrics(evaluations); } diff --git a/src/services/diff-vector-store.service.ts b/src/services/diff-vector-store.service.ts index a4fe40a..699131c 100644 --- a/src/services/diff-vector-store.service.ts +++ b/src/services/diff-vector-store.service.ts @@ -145,7 +145,17 @@ export class DiffVectorStoreService { const docData = this.parseDiffIntoDocuments(commitDiff); if (docData.length === 0) { - throw new Error('No documents extracted from diff'); + console.log('āš ļø No documents extracted from diff - likely empty or metadata-only commit'); + //TODO: check if this is correct + // Create a minimal document to avoid breaking the system + docData.push({ + content: 'Empty or metadata-only commit', + metadata: { + type: 'empty-commit', + file: 'N/A', + commitTag: this.commitTag, + }, + }); } // Build vocabulary from all content diff --git a/src/services/metrics-calculation.service.ts b/src/services/metrics-calculation.service.ts index 8c833b6..db66384 100644 --- a/src/services/metrics-calculation.service.ts +++ b/src/services/metrics-calculation.service.ts @@ -4,18 +4,14 @@ import * as fs from 'fs/promises'; import * as path from 'path'; +import * as _ from 'lodash'; import { AgentResult } from '../agents/agent.interface'; - -export interface MetricScores { - functionalImpact: number; - idealTimeHours: number; - testCoverage: number; - codeQuality: number; - codeComplexity: number; - actualTimeHours: number; - technicalDebtHours: number; - debtReductionHours: number; -} +import { + calculateWeightedAverage, + SEVEN_PILLARS, + PillarName, +} from '../constants/agent-weights.constants'; +import { MetricScores } from 'types/output.types'; export interface AuthorMetrics { commits: number; @@ -25,8 +21,32 @@ export interface AuthorMetrics { impact: number; time: number; techDebt: number; + commitScore: number; + baciScore?: number; +} + +export interface BaciDataPoint { + commits: number; + baseTCS: number; +} + +export interface BaciDefaults { + qualityPrior: number; + shrinkageStrength: number; + volumeSensitivity: number; + minPrVariationPct: number; + sigmoidSteepness: number; } +// BACI default configuration +export const BACI_DEFAULTS: BaciDefaults = { + qualityPrior: 5.5, // Bayesian prior for quality (1-10 scale) + shrinkageStrength: 3.0, // Shrinkage parameter for small samples + volumeSensitivity: 0.3, // Sensitivity to commit volume variations + minPrVariationPct: 0.2, // Minimum RSD to apply volume adjustments + sigmoidSteepness: 0.8, // Sigmoid steepness for normalization +}; + /** * Centralized service for all metrics calculations * Eliminates duplication and provides single source of truth @@ -41,22 +61,14 @@ export class MetricsCalculationService { return {}; } - // Import weight calculation function and constants - const { - calculateWeightedAverage, - SEVEN_PILLARS, - } = require('../constants/agent-weights.constants'); - // Get final round agents (last 5 entries) - should be 1 per agent role const finalAgents = agentResults.slice(-5); - // Metric names for weighted calculation - const metricNames = SEVEN_PILLARS as unknown as (keyof MetricScores)[]; - // Calculate weighted average for each metric - const averagedMetrics: Partial = {}; + const averagedMetrics: Record = {}; - metricNames.forEach((metricName: keyof MetricScores) => { + // Process each pillar metric (excluding commitScore which is calculated) + SEVEN_PILLARS.forEach((metricName: PillarName) => { const contributors: Array<{ agentName: string; score: number | null }> = []; finalAgents.forEach((agent: any) => { @@ -70,22 +82,39 @@ export class MetricsCalculationService { }); if (contributors.length > 0) { - const weightedValue = calculateWeightedAverage(contributors, metricName); + const weightedValue = calculateWeightedAverage(contributors, metricName as PillarName); // Determine decimal places based on metric if (metricName.includes('Hours') || metricName.includes('Time')) { - averagedMetrics[metricName] = Number(weightedValue.toFixed(2)) as any; + averagedMetrics[metricName] = Number(weightedValue?.toFixed(2)); } else { - averagedMetrics[metricName] = Number(weightedValue.toFixed(1)) as any; + averagedMetrics[metricName] = Number(weightedValue?.toFixed(1)); } } }); - return averagedMetrics; + // Calculate commitScore from the weighted metrics + if ( + typeof averagedMetrics.codeQuality === 'number' && + typeof averagedMetrics.codeComplexity === 'number' && + typeof averagedMetrics.actualTimeHours === 'number' && + typeof averagedMetrics.idealTimeHours === 'number' + ) { + averagedMetrics.commitScore = Number( + this.calculateCommitScoreFromMetrics( + averagedMetrics.codeQuality, + averagedMetrics.codeComplexity, + averagedMetrics.actualTimeHours, + averagedMetrics.idealTimeHours + ).toFixed(1) + ); + } + + return averagedMetrics as Partial; } /** * Calculate simple average metrics from multiple evaluations - * Used for author statistics aggregation + * Used for author statistics aggregation - uses weighted consensus values */ static calculateSimpleAverageMetrics(evaluations: any[]): AuthorMetrics { const stats: AuthorMetrics = { @@ -96,19 +125,30 @@ export class MetricsCalculationService { impact: 0, time: 0, techDebt: 0, + commitScore: 0, }; let validMetrics = 0; + let commitScoreSum = 0; + for (const evalData of evaluations) { - // Extract metrics from the last agent result (consensus) - const lastAgent = evalData.agents[evalData.agents.length - 1]; - if (lastAgent && lastAgent.metrics) { - stats.quality += lastAgent.metrics.codeQuality || 0; - stats.complexity += lastAgent.metrics.codeComplexity || 0; - stats.tests += lastAgent.metrics.testCoverage || 0; - stats.impact += lastAgent.metrics.functionalImpact || 0; - stats.time += lastAgent.metrics.actualTimeHours || 0; - stats.techDebt += lastAgent.metrics.technicalDebtHours || 0; + // Use weighted consensus metrics for consistency + const metrics = this.calculateWeightedMetrics(evalData.agents); + + if (metrics && Object.keys(metrics).length > 0) { + const quality = metrics.codeQuality || 0; + const complexity = metrics.codeComplexity || 0; + const actualTime = metrics.actualTimeHours || 0; + const commitScore = metrics.commitScore || 0; // Should be calculated by calculateWeightedMetrics + + stats.quality += quality; + stats.complexity += complexity; + stats.tests += metrics.testCoverage || 0; + stats.impact += metrics.functionalImpact || 0; + stats.time += actualTime; + const netDebt = (metrics.technicalDebtHours || 0) - (metrics.debtReductionHours || 0); + stats.techDebt += netDebt; + commitScoreSum += commitScore; validMetrics++; } } @@ -118,12 +158,14 @@ export class MetricsCalculationService { } // Calculate averages - stats.quality /= validMetrics; - stats.complexity /= validMetrics; - stats.tests /= validMetrics; - stats.impact /= validMetrics; - stats.time /= validMetrics; + stats.quality = Number((stats.quality / validMetrics).toFixed(1)); + stats.complexity = Number((stats.complexity / validMetrics).toFixed(1)); + stats.tests = Number((stats.tests / validMetrics).toFixed(1)); + stats.impact = Number((stats.impact / validMetrics).toFixed(1)); + stats.time = Number((stats.time / validMetrics).toFixed(2)); + stats.commitScore = Number((commitScoreSum / validMetrics).toFixed(1)); // Tech debt is total, not average + stats.techDebt = Number(stats.techDebt.toFixed(2)); return stats; } @@ -157,4 +199,373 @@ export class MetricsCalculationService { const resultsPath = path.join(evaluationDir, 'results.json'); return this.loadMetricsFromFile(resultsPath); } + + /** + * Calculate BACI (Bayesian Author Code Intelligence) scores + * Uses Bayesian inference with volume sensitivity for normalized developer scoring + */ + static computeBaciBoundedInteractive( + data: BaciDataPoint[], + options?: Partial + ): number[] { + if (!data || data.length === 0) { + return []; + } + + // Merge defaults with provided options + const config = { ...BACI_DEFAULTS, ...options }; + const { + qualityPrior, + shrinkageStrength, + volumeSensitivity, + minPrVariationPct, + sigmoidSteepness, + } = config; + const muQ = qualityPrior; + const kappa = shrinkageStrength; + + // Calculate base quality with Bayesian shrinkage + const baseQuality = data.map((point) => { + const n = point.commits; + const alpha = n / (n + kappa); + return alpha * point.baseTCS + (1 - alpha) * muQ; + }); + + // Calculate effort statistics + const commitCounts = data.map((point) => point.commits); + const muEffort = this.mean(commitCounts); + const sigmaEffort = this.std(commitCounts); + + // Calculate relative standard deviation (RSD) + const rsd = muEffort <= 0 ? Number.POSITIVE_INFINITY : sigmaEffort / muEffort; + + // Calculate volume multiplier + let volumeMult: number[]; + if (rsd < minPrVariationPct) { + volumeMult = data.map(() => 1.0); + } else { + volumeMult = data.map((point) => { + const effortZ = sigmaEffort === 0 ? 0 : (point.commits - muEffort) / sigmaEffort; + const clippedEffortZ = this.clip(effortZ, -2.0, 2.0); + return 1.0 + volumeSensitivity * Math.tanh(clippedEffortZ); + }); + } + + // Calculate raw BACI scores + const baciRaw = baseQuality.map((base, index) => base * volumeMult[index]); + + // Calculate team median + const teamMedian = this.median(baciRaw); + + // Apply sigmoid normalization + const beta = sigmoidSteepness; + const sigmoidNorm = baciRaw.map((raw) => 1 / (1 + Math.exp(-beta * (raw - teamMedian)))); + + // Final BACI scores bounded between 1-10 + const baciFinal = sigmoidNorm.map((norm) => 1.0 + 8.99 * norm); + + return baciFinal; + } + + /** + * Calculate user BACI scores from aggregated metrics + * Integrates with existing user metrics calculation pipeline + */ + static calculateUserBaciScores( + userBaseTCSMap: Record, + userCommitCounts: Record, + options?: Partial + ): Record { + // Convert data to BACI data points format + const dataPoints: BaciDataPoint[] = Object.keys(userBaseTCSMap).map((user) => ({ + commits: userCommitCounts[user] || 0, + baseTCS: userBaseTCSMap[user] || 0, + })); + + if (dataPoints.length === 0) { + return {}; + } + + // Calculate BACI scores + const baciScores = this.computeBaciBoundedInteractive(dataPoints, options); + + // Map scores back to users + const result: Record = {}; + const users = Object.keys(userBaseTCSMap); + users.forEach((user, index) => { + if (index < baciScores.length) { + result[user] = Math.round(baciScores[index] * 10) / 10; // Round to 1 decimal place + } + }); + + return result; + } + + /** + * Complete BACI calculation pipeline from raw metrics + * This is the main entry point for BACI scoring + */ + static calculateTeamBaciScores( + metrics: any[], // AggregatedPrMetricsViewEntity[] + options?: Partial + ): Record { + // Step 1: Calculate base TCS scores from metrics + const userBaseTCSMap = this.calculateUserBaseTCS(metrics); + + // Step 2: Calculate commit counts per user + const userCommitCounts: Record = {}; + metrics.forEach((metric) => { + const user = metric.createdBy; + if (user) { + userCommitCounts[user] = (userCommitCounts[user] || 0) + 1; + } + }); + + // Step 3: Calculate BACI scores + return this.calculateUserBaciScores(userBaseTCSMap, userCommitCounts, options); + } + + /** + * Calculate enhanced author statistics with commit scores + * This method integrates commit score calculations into comprehensive stats + */ + static calculateEnhancedAuthorStats( + metrics: any[] // Raw metrics data + ): Record { + const groupedByUser = _.groupBy(metrics, 'createdBy'); + + return _.mapValues(groupedByUser, (userMetrics: any[], user: string) => { + const avgMetrics = this.calculateAverageMetrics(userMetrics); + + // Calculate base stats + const stats: AuthorMetrics = { + commits: userMetrics.length, + quality: avgMetrics.avgTestingQuality || 0, // Using testing quality as a proxy + complexity: + userMetrics.reduce((sum, m) => sum + (m.codeComplexity || m.complexityScore || 0), 0) / + userMetrics.length, + tests: avgMetrics.avgTestingQuality || 0, + impact: avgMetrics.avgFunctionalImpact || 0, + time: + userMetrics.reduce((sum, m) => sum + (m.actualTimeHours || m.timeHours || 0), 0) / + userMetrics.length, + techDebt: userMetrics.reduce( + (sum, m) => sum + (m.technicalDebtHours || m.techDebtHours || 0), + 0 + ), + commitScore: avgMetrics.avgCommitScore || 0, + }; + + // Calculate BACI score if we have enough data + let baciScore: number | undefined; + try { + const userBaseTCSMap = { [user]: this.calculateUserBaseTCS(userMetrics)[user] || 0 }; + const userCommitCounts = { [user]: userMetrics.length }; + const baciResult = this.calculateUserBaciScores(userBaseTCSMap, userCommitCounts); + baciScore = baciResult[user]; + } catch (error) { + // BACI calculation failed, that's okay + baciScore = undefined; + } + + return { ...stats, baciScore }; + }); + } + + /** + * Calculate team summary with commit scores and BACI scores + */ + static calculateTeamSummary(metrics: any[]): { + teamStats: Record; + teamBaci: Record; + } { + const teamStats = this.calculateEnhancedAuthorStats(metrics); + const teamBaci = this.calculateTeamBaciScores(metrics); + + Object.keys(teamStats).forEach((user) => { + if (teamBaci[user] !== undefined) { + teamStats[user].baciScore = teamBaci[user]; + } + }); + + return { + teamStats, + teamBaci, + }; + } + + /** + * Calculate user base TCS (Technical Contribution Score) from aggregated metrics + * This is the base quality score used in BACI calculation + */ + private static calculateUserBaseTCS(metrics: any[]): Record { + const groupedByUser = _.groupBy(metrics, 'createdBy'); + return _.mapValues(groupedByUser, (userMetrics: any[]) => { + const metricsValues = this.calculateAverageMetrics(userMetrics); + // Calculate base TCS as weighted average of key metrics + const scores = [ + metricsValues.avgCommitScore || 0, + metricsValues.avgTestingQuality || 0, + metricsValues.avgTechnicalDebtRate || 0, + metricsValues.avgDeliveryRate || 0, + metricsValues.avgFunctionalImpact || 0, + ].filter((score) => score > 0); // Only include non-zero scores + + return scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0; + }); + } + + /** + * Calculate commit score from individual metrics + * Public method for use by external modules + */ + static calculateCommitScoreFromMetrics( + codeQuality: number, + codeComplexity: number, + actualTimeHours: number, + idealTimeHours: number + ): number { + // Calculate normalized estimation + const normalizedEstimation = + idealTimeHours > 0 + ? Math.max(0, 10 - (Math.abs(actualTimeHours - idealTimeHours) / idealTimeHours) * 10) + : 5; + + // Calculate penalty + const actualTimeMinutes = actualTimeHours * 60; + const penalty = this.calculatePenalty(actualTimeMinutes, codeComplexity, codeQuality); + + // Calculate commit score + return this.calculateCommitScore(codeQuality, codeComplexity, normalizedEstimation, penalty); + } + + /** + * Calculate commit score using quality, complexity, estimation, and penalty factors + */ + private static calculateCommitScore( + qualityScore: number, + complexityScore: number, + normalizedEstimation: number, + penalty: number + ): number { + let score = + qualityScore * 0.4 - complexityScore * 0.3 + normalizedEstimation * 0.3 + 3 - penalty; + score = Math.max(1, Math.min(10, score)); + return score; + } + + /** + * Calculate penalty based on actual time, complexity, and quality + */ + private static calculatePenalty( + actualTimeMinutes: number, + complexityScore: number, + qualityScore: number + ): number { + const timeFactor = 1 / (1 + (actualTimeMinutes / 60) ** 2); + const complexityPenalty = (complexityScore / 10) ** 2 * timeFactor * 4; + const qualityPenalty = ((10 - qualityScore) / 10) ** 2 * timeFactor * 4; + const totalPenalty = Math.min(4, Math.max(complexityPenalty, qualityPenalty)); + return totalPenalty; + } + + /** + * Helper method to calculate average metrics from user's metric array + * Dynamically calculates commit scores if not provided in raw data + */ + private static calculateAverageMetrics(userMetrics: any[]): { + avgCommitScore?: number; + avgTestingQuality?: number; + avgTechnicalDebtRate?: number; + avgDeliveryRate?: number; + avgFunctionalImpact?: number; + } { + if (userMetrics.length === 0) { + return {}; + } + + const sums = { + commitScore: 0, + testingQuality: 0, + technicalDebtRate: 0, + deliveryRate: 0, + functionalImpact: 0, + }; + + let validCount = 0; + userMetrics.forEach((metric) => { + // Calculate or use existing commit score + let commitScore = metric.commitScore; + if (commitScore === undefined) { + // Dynamic calculation if commit score not provided + const quality = metric.codeQuality || metric.qualityScore || 5; + const complexity = metric.codeComplexity || metric.complexityScore || 5; + const actualTime = metric.actualTimeHours || metric.timeHours || 1; + const idealTime = metric.idealTimeHours || actualTime; + + commitScore = this.calculateCommitScoreFromMetrics( + quality, + complexity, + actualTime, + idealTime + ); + } + + // Adapt these field names based on your actual metric structure + sums.commitScore += commitScore; + if (metric.testingQuality !== undefined) sums.testingQuality += metric.testingQuality; + if (metric.technicalDebtRate !== undefined) + sums.technicalDebtRate += metric.technicalDebtRate; + if (metric.deliveryRate !== undefined) sums.deliveryRate += metric.deliveryRate; + if (metric.functionalImpact !== undefined) sums.functionalImpact += metric.functionalImpact; + validCount++; + }); + + if (validCount === 0) { + return {}; + } + + return { + avgCommitScore: sums.commitScore / validCount, + avgTestingQuality: sums.testingQuality / validCount, + avgTechnicalDebtRate: sums.technicalDebtRate / validCount, + avgDeliveryRate: sums.deliveryRate / validCount, + avgFunctionalImpact: sums.functionalImpact / validCount, + }; + } + + /** + * Calculate mean of an array + */ + private static mean(arr: number[]): number { + if (arr.length === 0) return 0; + return arr.reduce((a, b) => a + b, 0) / arr.length; + } + + /** + * Calculate standard deviation of an array + */ + private static std(arr: number[]): number { + if (arr.length === 0) return 0; + const m = this.mean(arr); + const variance = arr.reduce((s, x) => s + (x - m) ** 2, 0) / arr.length; + return Math.sqrt(variance); + } + + /** + * Calculate median of an array + */ + private static median(arr: number[]): number { + if (arr.length === 0) return 0; + const sorted = [...arr].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; + } + + /** + * Clip a value between min and max bounds + */ + private static clip(x: number, minVal: number, maxVal: number): number { + return Math.max(minVal, Math.min(maxVal, x)); + } } diff --git a/src/types/output.types.ts b/src/types/output.types.ts index 97512d0..9684d29 100644 --- a/src/types/output.types.ts +++ b/src/types/output.types.ts @@ -16,9 +16,9 @@ export interface TokenSnapshot { } /** - * Metrics snapshot for a specific evaluation + * Metrics scores for a specific evaluation */ -export interface MetricsSnapshot { +export interface MetricScores { functionalImpact: number; idealTimeHours: number; testCoverage: number; @@ -27,8 +27,8 @@ export interface MetricsSnapshot { actualTimeHours: number; technicalDebtHours: number; debtReductionHours: number; + commitScore: number; } - /** * Single entry in evaluation history * Stores all important metrics from a re-evaluation @@ -37,7 +37,7 @@ export interface EvaluationHistoryEntry { timestamp: string; source: string; evaluationNumber: number; - metrics: MetricsSnapshot; + metrics: MetricScores; tokens: TokenSnapshot; convergenceScore: number; } diff --git a/test/baci-integration.test.ts b/test/baci-integration.test.ts new file mode 100644 index 0000000..e4ddc86 --- /dev/null +++ b/test/baci-integration.test.ts @@ -0,0 +1,252 @@ +// test/baci-integration.test.ts +// Test file for BACI integration in MetricsCalculationService + +import { + MetricsCalculationService, + BaciDataPoint, + BACI_DEFAULTS, +} from '../src/services/metrics-calculation.service'; + +describe('BACI Integration Tests', () => { + describe('computeBaciBoundedInteractive', () => { + it('should calculate BACI scores for sample data with commits', () => { + const testData: BaciDataPoint[] = [ + { commits: 5, baseTCS: 6.5 }, + { commits: 10, baseTCS: 7.2 }, + { commits: 3, baseTCS: 5.8 }, + { commits: 8, baseTCS: 7.0 }, + { commits: 12, baseTCS: 8.1 }, + ]; + + const baciScores = MetricsCalculationService.computeBaciBoundedInteractive(testData); + + // Validate results + expect(baciScores).toHaveLength(5); + expect(baciScores.every((score) => score >= 1.0 && score <= 10.0)).toBe(true); + + // Test with custom options + const customOptions = { + qualityPrior: 6.0, + volumeSensitivity: 0.4, + }; + const customBaciScores = MetricsCalculationService.computeBaciBoundedInteractive( + testData, + customOptions + ); + expect(customBaciScores).toHaveLength(5); + }); + + it('should handle edge cases', () => { + // Empty data + expect(MetricsCalculationService.computeBaciBoundedInteractive([])).toEqual([]); + + // Single data point + const singlePoint = [{ commits: 5, baseTCS: 6.5 }]; + const result = MetricsCalculationService.computeBaciBoundedInteractive(singlePoint); + expect(result).toHaveLength(1); + expect(result[0]).toBeGreaterThanOrEqual(1.0); + expect(result[0]).toBeLessThanOrEqual(10.0); + }); + }); + + describe('calculateUserBaciScores', () => { + it('should calculate user BACI scores from TCS and commit data', () => { + const userBaseTCSMap = { + alice: 7.2, + bob: 6.5, + charlie: 8.1, + diana: 5.8, + }; + + const userCommitCounts = { + alice: 10, + bob: 5, + charlie: 12, + diana: 3, + }; + + const baciScores = MetricsCalculationService.calculateUserBaciScores( + userBaseTCSMap, + userCommitCounts + ); + + // Validate results + expect(Object.keys(baciScores)).toEqual(['alice', 'bob', 'charlie', 'diana']); + Object.values(baciScores).forEach((score) => { + expect(score).toBeGreaterThanOrEqual(1.0); + expect(score).toBeLessThanOrEqual(10.0); + }); + }); + + it('should handle empty input', () => { + const result = MetricsCalculationService.calculateUserBaciScores({}, {}); + expect(result).toEqual({}); + }); + }); + + describe('calculateTeamBaciScores', () => { + it('should calculate complete BACI pipeline from raw metrics', () => { + const mockMetrics = [ + { + createdBy: 'alice', + commitScore: 8.5, + testingQuality: 7.0, + technicalDebtRate: 6.5, + deliveryRate: 8.0, + functionalImpact: 7.5, + }, + { + createdBy: 'alice', + commitScore: 7.5, + testingQuality: 8.0, + technicalDebtRate: 7.0, + deliveryRate: 7.5, + functionalImpact: 8.0, + }, + { + createdBy: 'bob', + commitScore: 6.0, + testingQuality: 6.5, + technicalDebtRate: 5.5, + deliveryRate: 6.0, + functionalImpact: 6.5, + }, + { + createdBy: 'bob', + commitScore: 6.5, + testingQuality: 6.0, + technicalDebtRate: 6.0, + deliveryRate: 6.5, + functionalImpact: 6.0, + }, + ]; + + const baciScores = MetricsCalculationService.calculateTeamBaciScores(mockMetrics); + + // Validate results + expect(baciScores).toHaveProperty('alice'); + expect(baciScores).toHaveProperty('bob'); + expect(typeof baciScores.alice).toBe('number'); + expect(typeof baciScores.bob).toBe('number'); + + // All scores should be in valid range + Object.values(baciScores).forEach((score) => { + expect(score).toBeGreaterThanOrEqual(1.0); + expect(score).toBeLessThanOrEqual(10.0); + }); + + // Alice should have higher BACI score due to better metrics and more commits + expect(baciScores.alice).toBeGreaterThan(baciScores.bob); + }); + }); + + describe('calculateSimpleAverageMetrics with commitScore', () => { + it('should calculate author metrics including commit scores', () => { + const mockEvaluations = [ + { + agents: [ + {}, + {}, + {}, + {}, // Other agents + { + // Last agent (consensus) + metrics: { + codeQuality: 8.0, + codeComplexity: 6.0, + testCoverage: 7.5, + functionalImpact: 8.2, + actualTimeHours: 2.0, + idealTimeHours: 1.8, + technicalDebtHours: 1.2, + }, + }, + ], + }, + ]; + + const stats = MetricsCalculationService.calculateSimpleAverageMetrics(mockEvaluations); + + expect(stats.commits).toBe(1); + expect(stats.commitScore).toBeGreaterThan(0); + expect(stats.commitScore).toBeLessThanOrEqual(10); + expect(stats.quality).toBe(8.0); + expect(stats.complexity).toBe(6.0); + }); + }); + + describe('calculateEnhancedAuthorStats', () => { + it('should calculate enhanced stats with BACI scores', () => { + const mockMetrics = [ + { + createdBy: 'alice', + codeQuality: 8.0, + codeComplexity: 5.0, + actualTimeHours: 2.0, + testingQuality: 7.5, + functionalImpact: 8.0, + }, + { + createdBy: 'bob', + codeQuality: 6.0, + codeComplexity: 7.0, + actualTimeHours: 3.0, + testingQuality: 6.0, + functionalImpact: 6.5, + }, + ]; + + const enhancedStats = MetricsCalculationService.calculateEnhancedAuthorStats(mockMetrics); + + expect(enhancedStats).toHaveProperty('alice'); + expect(enhancedStats).toHaveProperty('bob'); + expect(enhancedStats.alice.commitScore).toBeGreaterThan(0); + expect(enhancedStats.alice.baciScore).toBeDefined(); + expect(enhancedStats.bob.commitScore).toBeGreaterThan(0); + }); + }); + + describe('calculateTeamSummary', () => { + it('should provide comprehensive team analysis with rankings', () => { + const mockMetrics = [ + { + createdBy: 'alice', + codeQuality: 8.5, + codeComplexity: 5.0, + actualTimeHours: 2.0, + testingQuality: 8.0, + functionalImpact: 7.5, + }, + { + createdBy: 'alice', + codeQuality: 7.5, + codeComplexity: 6.0, + actualTimeHours: 1.5, + testingQuality: 7.5, + functionalImpact: 8.0, + }, + { + createdBy: 'bob', + codeQuality: 6.0, + codeComplexity: 7.0, + actualTimeHours: 3.0, + testingQuality: 6.0, + functionalImpact: 6.0, + }, + ]; + + const summary = MetricsCalculationService.calculateTeamSummary(mockMetrics); + + expect(summary).toHaveProperty('teamStats'); + expect(summary).toHaveProperty('teamBaci'); + // Note: rankings were removed from calculateTeamSummary as they weren't used in production + + // Verify team stats and BACI scores are populated + expect(Object.keys(summary.teamStats)).toHaveLength(2); + expect(Object.keys(summary.teamBaci)).toHaveLength(2); + + // Alice should have higher BACI score due to better metrics and more commits + expect(summary.teamBaci.alice).toBeGreaterThan(summary.teamBaci.bob); + }); + }); +});