[FEATURE] Added a "Never Worked" state to the tests

migrations/versions/d9f4e5a6b7c8_.py

@@ -0,0 +1,27 @@
+"""Add never_worked column to regression_test table
+
+Revision ID: d9f4e5a6b7c8
+Revises: c8f3a2b1d4e5
+Create Date: 2026-03-20 23:25:21.411651000000
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'd9f4e5a6b7c8'
+down_revision = 'c8f3a2b1d4e5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('regression_test', sa.Column('never_worked', sa.Boolean(), nullable=False, server_default='false'))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('regression_test', 'never_worked')
+    # ### end Alembic commands ###

mod_ci/controllers.py

@@ -45,7 +45,8 @@
 from mod_sample.models import Issue
 from mod_test.controllers import get_test_results
 from mod_test.models import (Fork, Test, TestPlatform, TestProgress,
-                             TestResult, TestResultFile, TestStatus, TestType)
+                             TestResult, TestResultFile, TestResultStatus,
+                             TestStatus, TestType)
 from utility import is_valid_signature, request_from_github
 
 # Timeout constants (in seconds)
@@ -2760,6 +2761,7 @@ def get_info_for_pr_comment(test: Test) -> PrCommentInfo:
     extra_failed_tests = []
     common_failed_tests = []
     fixed_tests = []
+    never_worked_tests = []
     category_stats = []
 
     test_results = get_test_results(test)
@@ -2768,20 +2770,26 @@ def get_info_for_pr_comment(test: Test) -> PrCommentInfo:
         category_name = category_results['category'].name
 
         category_test_pass_count = 0
-        for test in category_results['tests']:
-            if not test['error']:
+        for t in category_results['tests']:
+            if not t['error']:
                 category_test_pass_count += 1
-                if last_test_master and getattr(test['test'], platform_column) != last_test_master.id:
-                    fixed_tests.append(test['test'])
+                if last_test_master and getattr(t['test'], platform_column) != last_test_master.id:
+                    fixed_tests.append(t['test'])
             else:
-                if last_test_master and getattr(test['test'], platform_column) != last_test_master.id:
-                    common_failed_tests.append(test['test'])
+                # Separate out tests that have NEVER passed on any CCExtractor version
+                if t['status'] == TestResultStatus.never_worked:
+                    never_worked_tests.append(t['test'])
+                elif last_test_master and getattr(t['test'], platform_column) != last_test_master.id:
+                    common_failed_tests.append(t['test'])
                 else:
-                    extra_failed_tests.append(test['test'])
+                    extra_failed_tests.append(t['test'])
 
         category_stats.append(CategoryTestInfo(category_name, len(category_results['tests']), category_test_pass_count))
 
-    return PrCommentInfo(category_stats, extra_failed_tests, fixed_tests, common_failed_tests, last_test_master)
+    return PrCommentInfo(
+        category_stats, extra_failed_tests, fixed_tests, common_failed_tests,
+        last_test_master, never_worked_tests
+    )
 
 
 def comment_pr(test: Test) -> str:
@@ -2817,6 +2825,9 @@ def comment_pr(test: Test) -> str:
         log.debug(f"GitHub PR Comment ID {comment.id} Uploaded for Test_id: {test_id}")
     except Exception as e:
         log.error(f"GitHub PR Comment Failed for Test_id: {test_id} with Exception {e}")
+
+    # Determine PR status:
+    # SUCCESS if no regressions caused by PR (never_worked tests don't count)
     return Status.SUCCESS if len(comment_info.extra_failed_tests) == 0 else Status.FAILURE
 
 

mod_ci/models.py

@@ -179,3 +179,4 @@ class PrCommentInfo:
     fixed_tests: List[RegressionTest]
     common_failed_tests: List[RegressionTest]
     last_test_master: Test
+    never_worked_tests: List[RegressionTest]

mod_regression/controllers.py

@@ -160,6 +160,7 @@ def test_edit(regression_id):
         test.input_type = InputType.from_string(form.input_type.data)
         test.output_type = OutputType.from_string(form.output_type.data)
         test.description = form.description.data
+        test.never_worked = form.never_worked.data
 
         g.db.commit()
         g.log.info(f'regression test with id: {regression_id} updated!')
@@ -174,6 +175,7 @@ def test_edit(regression_id):
         form.input_type.data = test.input_type.value
         form.output_type.data = test.output_type.value
         form.description.data = test.description
+        form.never_worked.data = test.never_worked
 
     return {'form': form, 'regression_id': regression_id}
 
@@ -247,6 +249,7 @@ def test_add():
             input_type=InputType.from_string(form.input_type.data),
             output_type=OutputType.from_string(form.output_type.data),
             description=form.description.data,
+            never_worked=form.never_worked.data
         )
         g.db.add(new_test)
         category = Category.query.filter(Category.id == form.category_id.data).first()

mod_regression/forms.py

@@ -1,8 +1,8 @@
 """Maintain forms related to CRUD operations on regression tests."""
 
 from flask_wtf import FlaskForm
-from wtforms import (HiddenField, IntegerField, SelectField, StringField,
-                     SubmitField, TextAreaField)
+from wtforms import (BooleanField, HiddenField, IntegerField, SelectField,
+                     StringField, SubmitField, TextAreaField)
 from wtforms.validators import DataRequired, InputRequired, Length
 
 from mod_regression.models import InputType, OutputType
@@ -36,6 +36,7 @@ class CommonTestForm(FlaskForm):
     )
     category_id = SelectField("Category", coerce=int)
     expected_rc = IntegerField("Expected Runtime Code", [InputRequired(message="Expected Runtime Code can't be empty")])
+    never_worked = BooleanField("Never Worked", default=False)
 
 
 class AddTestForm(CommonTestForm):

mod_regression/models.py

@@ -97,9 +97,10 @@ class RegressionTest(Base):
     last_passed_on_windows = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL"))
     last_passed_on_linux = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL"))
     description = Column(String(length=1024))
+    never_worked = Column(Boolean(), default=False, nullable=False, server_default='false')
 
     def __init__(self, sample_id, command, input_type, output_type, category_id, expected_rc,
-                 active=True, description="") -> None:
+                 active=True, description="", never_worked=False) -> None:
         """
         Parametrized constructor for the RegressionTest model.
 
@@ -117,7 +118,10 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp
         :type expected_rc: int
         :param active: The value of the 'active' field of RegressionTest model
         :type active: bool
-
+        :param description: The value of the 'description' field of RegressionTest model
+        :type description: str
+        :param never_worked: Boolean flag whether the test has never worked for this sample
+        :type never_worked: bool
         """
         self.sample_id = sample_id
         self.command = command
@@ -127,6 +131,7 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp
         self.expected_rc = expected_rc
         self.active = active
         self.description = description
+        self.never_worked = never_worked
 
     def __repr__(self) -> str:
         """

mod_test/controllers.py

@@ -1,7 +1,7 @@
 """Logic to find all tests, their progress and details of individual test."""
 
 import os
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional, TypedDict
 
 from flask import (Blueprint, Response, abort, g, jsonify, redirect, request,
                    url_for)
@@ -17,9 +17,29 @@
 from mod_regression.models import (Category, RegressionTestOutput,
                                    regressionTestLinkTable)
 from mod_test.models import (Fork, Test, TestPlatform, TestProgress,
-                             TestResult, TestResultFile, TestStatus, TestType)
+                             TestResult, TestResultFile, TestResultStatus,
+                             TestStatus, TestType)
 from utility import serve_file_download
 
+
+class CategoryTestItem(TypedDict):
+    """Represent a single regression test item in a category."""
+
+    test: Any  # RegressionTest
+    result: Optional[TestResult]
+    files: List[TestResultFile]
+    error: bool
+    status: TestResultStatus
+
+
+class CategoryResult(TypedDict):
+    """Represent the test results for an entire category."""
+
+    category: Category
+    tests: List[CategoryTestItem]
+    error: bool
+
+
 mod_test = Blueprint('test', __name__)
 
 
@@ -54,9 +74,75 @@ def index():
     }
 
 
-def get_test_results(test) -> List[Dict[str, Any]]:
+def _get_files_for_test(test_id: int, rt_id: int) -> List[TestResultFile]:
+    """Fetch result files for a test."""
+    return TestResultFile.query.options(
+        joinedload(TestResultFile.regression_test_output).selectinload(
+            RegressionTestOutput.multiple_files
+        )
+    ).filter(
+        and_(TestResultFile.test_id == test_id, TestResultFile.regression_test_id == rt_id)
+    ).all()
+
+
+def _build_test_item(test: Test, rt: Any) -> CategoryTestItem:
+    """Build a single CategoryTestItem."""
+    result = next((r for r in test.results if r.regression_test_id == rt.id), None)
+    files = _get_files_for_test(test.id, rt.id)
+    return {
+        'test': rt,
+        'result': result,
+        'files': files,
+        'error': False,
+        'status': TestResultStatus.passed
+    }
+
+
+def _has_file_error(result_file: TestResultFile, result: Optional[TestResult]) -> bool:
+    """Evaluate if a single result file has an error."""
+    if result_file.got is None:
+        return False
+    if result is not None and result.exit_code != result.expected_rc:
+        return False
+
+    for file in result_file.regression_test_output.multiple_files:
+        if file.file_hashes == result_file.got:
+            return False
+    return True
+
+
+def _evaluate_category_test(category_test: CategoryTestItem) -> bool:
+    """Evaluate if a single regression test failed."""
+    test_error = False
+    result = category_test['result']
+
+    if result is not None and result.exit_code != result.expected_rc:
+        test_error = True
+
+    if category_test['files']:
+        for result_file in category_test['files']:
+            if _has_file_error(result_file, result):
+                test_error = True
+    else:
+        outputs = RegressionTestOutput.query.filter(and_(
+            RegressionTestOutput.regression_id == category_test['test'].id,
+            RegressionTestOutput.ignore.is_(False)
+        )).all()
+        got = None
+        if outputs:
+            test_error = True
+            got = 'error'
+        category_test['files'] = [TestResultFile(-1, -1, -1, '', got)]
+
+    return test_error
+
+
+def get_test_results(test) -> List[CategoryResult]:
     """
-    Get test results for each category.
+    Get test results for each category, with three-way pass/fail/never_worked classification.
+
+    The never_worked status is determined by the explicit `never_worked` boolean flag on each
+    RegressionTest, which is admin-editable from the regression test edit page.
 
     :param test: The test to retrieve the data for.
     :type test: Test
@@ -67,60 +153,39 @@ def get_test_results(test) -> List[Dict[str, Any]]:
     ).filter(
         Category.id.in_(populated_categories)
     ).order_by(Category.name.asc()).all()
-    results = [{
-        'category': category,
-        'tests': [{
-            'test': rt,
-            'result': next((r for r in test.results if r.regression_test_id == rt.id), None),
-            'files': TestResultFile.query.options(
-                joinedload(TestResultFile.regression_test_output).selectinload(
-                    RegressionTestOutput.multiple_files
-                )
-            ).filter(
-                and_(TestResultFile.test_id == test.id, TestResultFile.regression_test_id == rt.id)
-            ).all()
-        } for rt in category.regression_tests if rt.id in test.get_customized_regressiontests()]
-    } for category in categories]
-    # Run through the categories to see if they should be marked as failed or passed. A category failed if one or more
-    # tests in said category failed.
+    # Collect all regression test IDs that are part of this test run
+    all_rt_ids = set(test.get_customized_regressiontests())
+
+    results: List[CategoryResult] = []
+    for category in categories:
+        cat_tests = []
+        for rt in category.regression_tests:
+            if rt.id in all_rt_ids:
+                cat_tests.append(_build_test_item(test, rt))
+        results.append({
+            'category': category,
+            'tests': cat_tests,
+            'error': False
+        })
+
+    # Run through the categories to see if they should be marked as failed or passed.
     for category in results:
         error = False
         for category_test in category['tests']:
-            test_error = False
-            # A test fails if:
-            # - Exit code is not what we expected
-            # - There are result files but one of the files is [not identical
-            #   and not one of the multiple correct output files]
-            # - There are no result files but there should have been
-            result = category_test['result']
-            if result is not None and result.exit_code != result.expected_rc:
-                test_error = True
-            if len(category_test['files']) > 0:
-                for result_file in category_test['files']:
-                    if result_file.got is not None and (result is None or result.exit_code == result.expected_rc):
-                        file_error = True
-                        for file in result_file.regression_test_output.multiple_files:
-                            if file.file_hashes == result_file.got:
-                                file_error = False
-                                break
-                        test_error = file_error or test_error
-            else:
-                # We need to check if the regression test had any file that shouldn't have been ignored.
-                outputs = RegressionTestOutput.query.filter(and_(
-                    RegressionTestOutput.regression_id == category_test['test'].id,
-                    RegressionTestOutput.ignore.is_(False)
-                )).all()
-                got = None
-                if len(outputs) > 0:
-                    test_error = True
-                    got = 'error'
-                # Add dummy entry for pass/fail display
-                category_test['files'] = [TestResultFile(-1, -1, -1, '', got)]
-            # Store test status in error field
+            test_error = _evaluate_category_test(category_test)
             category_test['error'] = test_error
-            # Update category error
+
+            # --- Three-way classification: passed / failed / never_worked ---
+            if not test_error:
+                category_test['status'] = TestResultStatus.passed
+            elif category_test['test'].never_worked:
+                category_test['status'] = TestResultStatus.never_worked
+            else:
+                category_test['status'] = TestResultStatus.failed
+
             error = error or test_error
         category['error'] = error
+
     results.sort(key=lambda entry: entry['category'].name)
     return results