Feature/dimensionality column mean to be between (open-metadata#23984)

* Initial implementation for Dimensionality on Data Quality Tests

* Fix ColumnValuesToBeUnique and create TestCaseResult API

* Refactor dimension result

* Initial E2E Implementation without Impact Score

* Dimensionality Thin Slice

* Update generated TypeScript types

* Update generated TypeScript types

* Removed useless method to use the one we already had

* Fix Pandas Dimensionality checks

* Remove useless comments

* Implement PR comments, fix Tests

* Improve the code a bit

* Fix imports

* Implement Dimensionality for ColumnMeanToBeBetween

* Removed useless comments and improved minor things

* Implement UnitTests

* Fixes

* Moved import pandas to type checking

* Fix Min/Max being optional

* Fix Unittests

* small fixes

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: IceS2

ingestion/src/metadata/data_quality/validations/base_test_handler.py

@@ -62,6 +62,9 @@
 DIMENSION_IMPACT_SCORE_KEY = "impact_score"
 DIMENSION_FAILED_COUNT_KEY = "failed_count"
 DIMENSION_TOTAL_COUNT_KEY = "total_count"
+DIMENSION_SUM_VALUE_KEY = (
+    "sum_value"  # For statistical validators weighted calculations
+)
 
 
 class TestEvaluation(TypedDict, total=False):
@@ -208,6 +211,76 @@ def _run_dimensional_validation(self) -> List[DimensionResult]:
         """
         return []
 
+    def _get_test_parameters(self) -> Optional[dict]:
+        """Get test-specific parameters from test case
+
+        Default implementation returns None. Override in child classes
+        that need to extract and process test parameters.
+
+        Returns:
+            Optional[dict]: Test parameters, or None if validator has no parameters.
+        """
+        return None
+
+    def _evaluate_test_condition(
+        self, metric_values: dict, test_params: Optional[dict] = None
+    ) -> TestEvaluation:
+        """Evaluate the test condition based on computed metrics
+
+        This is the core logic that determines if the test passes or fails.
+        Override in child classes to implement test-specific evaluation logic.
+
+        Default implementation raises NotImplementedError. Validators that have been
+        migrated to the new pattern should override this method.
+
+        Args:
+            metric_values: Dictionary with Metrics enum names as keys
+                          e.g., {"COUNT": 100, "MEAN": 42.5}
+            test_params: Optional test parameters (bounds, allowed values, etc.)
+                        Some validators don't need parameters (e.g., uniqueness test)
+
+        Returns:
+            TestEvaluation: TypedDict with keys:
+                - matched: bool - whether test passed
+                - passed_rows: Optional[int] - number of passing rows (None for statistical tests)
+                - failed_rows: Optional[int] - number of failing rows (None for statistical tests)
+                - total_rows: Optional[int] - total row count (None for statistical tests)
+
+        Raises:
+            NotImplementedError: If child class doesn't override this method
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement _evaluate_test_condition()"
+        )
+
+    def _format_result_message(
+        self,
+        metric_values: dict,
+        dimension_info: Optional[DimensionInfo] = None,
+        test_params: Optional[dict] = None,
+    ) -> str:
+        """Format the result message for the test
+
+        Override in child classes to provide human-readable test results.
+
+        Default implementation raises NotImplementedError. Validators that have been
+        migrated to the new pattern should override this method.
+
+        Args:
+            metric_values: Dictionary with Metrics enum names as keys
+            dimension_info: Optional dimension details for dimensional results
+            test_params: Optional test parameters (for displaying bounds, thresholds, etc.)
+
+        Returns:
+            str: Formatted result message
+
+        Raises:
+            NotImplementedError: If child class doesn't override this method
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement _format_result_message()"
+        )
+
     def _extract_dimension_value(self, row: dict) -> str:
         """Extract and format dimension value from result row
 
@@ -278,6 +351,7 @@ def _create_dimension_result(
                 dimension_name=dimension_col_name,
                 dimension_value=dimension_value,
             ),
+            test_params=test_params,
         )
 
         test_result_values = self._get_test_result_values(metric_values)
@@ -443,8 +517,8 @@ def get_dimension_result_object(
         test_case_status: TestCaseStatus,
         result: str,
         test_result_value: List[TestResultValue],
-        total_rows: int,
-        passed_rows: int,
+        total_rows: Optional[int] = None,
+        passed_rows: Optional[int] = None,
         failed_rows: Optional[int] = None,
         impact_score: Optional[float] = None,
     ) -> "DimensionResult":
@@ -455,24 +529,30 @@ def get_dimension_result_object(
             test_case_status: Status of the test for this dimension combination
             result: Details of test case results for this dimension combination
             test_result_value: List of test result values
-            total_rows: Total number of rows in this dimension
-            passed_rows: Number of rows that passed for this dimension
-            failed_rows: Number of rows that failed for this dimension (auto-calculated if None)
+            total_rows: Total number of rows in this dimension (None for statistical validators)
+            passed_rows: Number of rows that passed for this dimension (None for statistical validators)
+            failed_rows: Number of rows that failed for this dimension (auto-calculated if None, None for statistical validators)
             impact_score: Optional impact score for this dimension (0-1 range)
 
         Returns:
             DimensionResult: Dimension result object with calculated percentages
         """
-        if failed_rows is None:
-            failed_rows = total_rows - passed_rows
-
-        # Derive one percentage from the other to ensure they sum to 100%
-        if total_rows > 0:
-            passed_rows_percentage = round(passed_rows / total_rows * 100, 2)
-            failed_rows_percentage = round(100 - passed_rows_percentage, 2)
+        # Handle row counts and percentages for statistical validators
+        if total_rows is None or passed_rows is None:
+            passed_rows_percentage = None
+            failed_rows_percentage = None
         else:
-            passed_rows_percentage = 0
-            failed_rows_percentage = 0
+            # Row-by-row validators: calculate percentages
+            if failed_rows is None:
+                failed_rows = total_rows - passed_rows
+
+            # Derive one percentage from the other to ensure they sum to 100%
+            if total_rows > 0:
+                passed_rows_percentage = round(passed_rows / total_rows * 100, 2)
+                failed_rows_percentage = round(100 - passed_rows_percentage, 2)
+            else:
+                passed_rows_percentage = 0
+                failed_rows_percentage = 0
 
         dimension_values_array = [
             DimensionValue(name=name, value=value)