PFM-TASK-6308 refactor: improve coverage evaluation logic and enhance reporting for project thresholds

Author: senthanal

tools/scripts/run-many/coverage-evaluator.ts

@@ -1,7 +1,18 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import * as core from '@actions/core';
-import { CoverageThreshold, getProjectThresholds, ThresholdConfig } from './threshold-handler';
+
+interface CoverageThreshold {
+  lines?: number;
+  statements?: number;
+  functions?: number;
+  branches?: number;
+}
+
+interface ThresholdConfig {
+  global: CoverageThreshold;
+  projects: Record<string, CoverageThreshold | null>;
+}
 
 interface CoverageSummary {
   lines: { pct: number };
@@ -20,23 +31,59 @@ interface ProjectCoverageResult {
     branches: number;
   } | null;
   status: 'PASSED' | 'FAILED' | 'SKIPPED';
-  failedMetrics?: string[];
+}
+
+/**
+ * Parses the COVERAGE_THRESHOLDS environment variable
+ */
+export function getCoverageThresholds(): ThresholdConfig {
+  if (!process.env.COVERAGE_THRESHOLDS) {
+    return { global: {}, projects: {} };
+  }
+
+  try {
+    return JSON.parse(process.env.COVERAGE_THRESHOLDS);
+  } catch (error) {
+    core.error(`Error parsing COVERAGE_THRESHOLDS: ${error.message}`);
+    return { global: {}, projects: {} };
+  }
+}
+
+/**
+ * Gets thresholds for a specific project
+ */
+function getProjectThresholds(project: string, thresholds: ThresholdConfig): CoverageThreshold | null {
+  // If project explicitly set to null, return null to skip
+  if (thresholds.projects && thresholds.projects[project] === null) {
+    return null;
+  }
+
+  // If project has specific thresholds, use those
+  if (thresholds.projects && thresholds.projects[project]) {
+    return thresholds.projects[project];
+  }
+
+  // Otherwise, use global thresholds if available
+  if (thresholds.global) {
+    return thresholds.global;
+  }
+
+  // If no thresholds defined, return null
+  return null;
 }
 
 /**
  * Evaluates coverage for all projects against their thresholds
  */
-export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig): boolean {
+export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig): { failedCount: number, projects: string[] } {
   if (!process.env.COVERAGE_THRESHOLDS) {
-    core.info('No coverage thresholds defined, skipping evaluation');
-    return true; // No thresholds defined, pass by default
+    return { failedCount: 0, projects: [] }; // No thresholds defined, pass by default
   }
 
-  let allProjectsPassed = true;
+  let failedCount = 0;
+  const failedProjects: string[] = [];
   const coverageResults: ProjectCoverageResult[] = [];
 
-  core.info(`Evaluating coverage for ${projects.length} projects`);
-
   for (const project of projects) {
     const projectThresholds = getProjectThresholds(project, thresholds);
 
@@ -60,46 +107,26 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig
         project,
         thresholds: projectThresholds,
         actual: null,
-        status: 'FAILED', // Mark as failed if no coverage report is found
-        failedMetrics: ['No coverage report found']
+        status: 'FAILED' // Mark as failed if no coverage report is found
       });
-      allProjectsPassed = false;
+      failedCount++;
+      failedProjects.push(project);
       continue;
     }
 
     try {
       const coverageData = JSON.parse(fs.readFileSync(coveragePath, 'utf8'));
       const summary = coverageData.total as CoverageSummary;
 
-      let projectPassed = true;
-      const failedMetrics: string[] = [];
-
-      // Check each metric if threshold is defined
-      if (projectThresholds.lines !== undefined && summary.lines.pct < projectThresholds.lines) {
-        projectPassed = false;
-        failedMetrics.push(`lines: ${summary.lines.pct.toFixed(2)}% < ${projectThresholds.lines}%`);
-      }
-
-      if (projectThresholds.statements !== undefined && summary.statements.pct < projectThresholds.statements) {
-        projectPassed = false;
-        failedMetrics.push(`statements: ${summary.statements.pct.toFixed(2)}% < ${projectThresholds.statements}%`);
-      }
-
-      if (projectThresholds.functions !== undefined && summary.functions.pct < projectThresholds.functions) {
-        projectPassed = false;
-        failedMetrics.push(`functions: ${summary.functions.pct.toFixed(2)}% < ${projectThresholds.functions}%`);
-      }
-
-      if (projectThresholds.branches !== undefined && summary.branches.pct < projectThresholds.branches) {
-        projectPassed = false;
-        failedMetrics.push(`branches: ${summary.branches.pct.toFixed(2)}% < ${projectThresholds.branches}%`);
-      }
+      const projectPassed =
+        (!projectThresholds.lines || summary.lines.pct >= projectThresholds.lines) &&
+        (!projectThresholds.statements || summary.statements.pct >= projectThresholds.statements) &&
+        (!projectThresholds.functions || summary.functions.pct >= projectThresholds.functions) &&
+        (!projectThresholds.branches || summary.branches.pct >= projectThresholds.branches);
 
       if (!projectPassed) {
-        core.error(`Project ${project} failed coverage thresholds: ${failedMetrics.join(', ')}`);
-        allProjectsPassed = false;
-      } else {
-        core.info(`Project ${project} passed all coverage thresholds`);
+        failedCount++;
+        failedProjects.push(project);
       }
 
       coverageResults.push({
@@ -111,8 +138,7 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig
           functions: summary.functions.pct,
           branches: summary.branches.pct
         },
-        status: projectPassed ? 'PASSED' : 'FAILED',
-        failedMetrics: projectPassed ? undefined : failedMetrics
+        status: projectPassed ? 'PASSED' : 'FAILED'
       });
     } catch (error) {
       core.error(`Error processing coverage for ${project}: ${error.message}`);
@@ -122,20 +148,21 @@ export function evaluateCoverage(projects: string[], thresholds: ThresholdConfig
         actual: null,
         status: 'FAILED'
       });
-      allProjectsPassed = false;
+      failedCount++;
+      failedProjects.push(project);
     }
   }
 
-  // Post results to PR comment
-  postCoverageComment(coverageResults);
+  // Post results to PR comment with special handling for single failing project
+  postCoverageComment(coverageResults, failedCount, failedProjects);
 
-  return allProjectsPassed;
+  return { failedCount, projects: failedProjects };
 }
 
 /**
  * Formats the coverage results into a markdown table
  */
-function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: string): string {
+function formatCoverageComment(results: ProjectCoverageResult[], failedCount: number, failedProjects: string[], artifactUrl: string): string {
   let comment = '## Test Coverage Results\n\n';
 
   if (results.length === 0) {
@@ -150,8 +177,7 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st
     if (result.status === 'SKIPPED') {
       comment += `| ${result.project} | All | N/A | N/A | ⏩ SKIPPED |\n`;
     } else if (result.actual === null) {
-      const errorMessage = result.failedMetrics ? result.failedMetrics[0] : 'No Data';
-      comment += `| ${result.project} | All | Defined | ${errorMessage} | ❌ FAILED |\n`;
+      comment += `| ${result.project} | All | Defined | No Data | ❌ FAILED |\n`;
     } else {
       const metrics = ['lines', 'statements', 'functions', 'branches'];
       metrics.forEach((metric, index) => {
@@ -160,8 +186,7 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st
 
         const threshold = result.thresholds[metric];
         const actual = result.actual[metric].toFixed(2);
-        const isMetricPassed = actual >= threshold;
-        const status = isMetricPassed ? '✅ PASSED' : '❌ FAILED';
+        const status = actual >= threshold ? '✅ PASSED' : '❌ FAILED';
 
         // Only include project name in the first row for this project
         const projectCell = index === 0 ? result.project : '';
@@ -171,23 +196,17 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st
     }
   });
 
-  // Add overall status
-  const overallStatus = results.every(r => r.status !== 'FAILED') ? '✅ PASSED' : '❌ FAILED';
-  comment += `\n### Overall Status: ${overallStatus}\n`;
-
-  // Add failed projects list if any
-  const failedProjects = results.filter(r => r.status === 'FAILED');
-  if (failedProjects.length > 0) {
-    comment += '\n### Failed Projects\n\n';
-    failedProjects.forEach(project => {
-      comment += `- **${project.project}**: `;
-      if (project.failedMetrics) {
-        comment += project.failedMetrics.join(', ');
-      } else {
-        comment += 'Failed to meet coverage thresholds';
-      }
-      comment += '\n';
-    });
+  // Add special message for single failing project
+  if (failedCount === 1) {
+    comment += `\n### Policy Exception: One project (${failedProjects[0]}) failed, but PR is allowed to pass ⚠️\n`;
+    comment += `According to team policy, PRs are allowed to pass when only one project fails the coverage threshold.\n`;
+  } else if (failedCount > 1) {
+    comment += `\n### Overall Status: ❌ FAILED\n`;
+    comment += `${failedCount} projects failed to meet coverage thresholds: ${failedProjects.join(', ')}\n`;
+    comment += `PRs are only allowed to pass when at most one project fails the coverage threshold.\n`;
+  } else {
+    comment += `\n### Overall Status: ✅ PASSED\n`;
+    comment += `All projects meet coverage thresholds! 🎉\n`;
   }
 
   // Add link to detailed HTML reports
@@ -201,11 +220,11 @@ function formatCoverageComment(results: ProjectCoverageResult[], artifactUrl: st
 /**
  * Writes the coverage results to a file for PR comment
  */
-function postCoverageComment(results: ProjectCoverageResult[]): void {
+function postCoverageComment(results: ProjectCoverageResult[], failedCount: number, failedProjects: string[]): void {
   // The actual artifact URL will be provided by GitHub Actions in the workflow
   const artifactUrl = process.env.COVERAGE_ARTIFACT_URL || '';
 
-  const comment = formatCoverageComment(results, artifactUrl);
+  const comment = formatCoverageComment(results, failedCount, failedProjects, artifactUrl);
 
   // Write to a file that will be used by thollander/actions-comment-pull-request action
   const gitHubCommentsFile = path.resolve(process.cwd(), 'coverage-report.txt');

tools/scripts/run-many/run-many.ts

@@ -79,22 +79,22 @@ function main() {
     cmd = getE2ECommand(cmd, base);
   }
 
-  if (areAffectedProjects) {
+  if (projects.length > 0) {
     runCommand(cmd);
 
     // Evaluate coverage if enabled and target is test
     if (coverageEnabled && target === 'test') {
       const thresholds = getCoverageThresholds();
+      const { failedCount, projects: failedProjects } = evaluateCoverage(projects, thresholds);
 
-      // Log the current coverage thresholds for debugging
-      core.info('Coverage threshold configuration:');
-      core.info(JSON.stringify(thresholds, null, 2));
-
-      const passed = evaluateCoverage(projects, thresholds);
-
-      if (!passed) {
-        core.setFailed('One or more projects failed to meet coverage thresholds');
+      // Continue running and only fail the build if more than one project fails
+      if (failedCount > 1) {
+        core.setFailed(`${failedCount} projects failed to meet coverage thresholds, which exceeds the limit of 1: ${failedProjects.join(', ')}`);
         process.exit(1);
+      } else if (failedCount === 1) {
+        core.warning(`1 project failed to meet coverage thresholds (${failedProjects[0]}), but we'll allow it to pass.`);
+      } else if (failedCount === 0) {
+        core.info('All projects passed coverage thresholds! 🎉');
       }
     }
   } else {
@@ -110,5 +110,3 @@ function main() {
     }
   }
 }
-
-main();