Fix regression testing false negative

The QoR parsing script returned -1 in case of a mismatch between arch file name in
the actual run and the golden results. If there was 1 run failures, the -1 would
cancel the 1 and as a result the regression tests would pass while having errors.
This commit fixes that by not throwing an exception in that specific case (Therefore
not returning -1) and taking the absolute value of the QoR and run failures.

Author: AmirhosseinPoolad

run_reg_test.py

@@ -199,8 +199,10 @@ def vtr_command_main(arg_list, prog=None):
         print("All tests passed")
     elif tests_run and total_num_func_failures != 0 or total_num_qor_failures != 0:
         print("Error: {} tests failed".format(total_num_func_failures + total_num_qor_failures))
-
-    sys.exit(total_num_func_failures + total_num_qor_failures)
+    
+    # If the QoR parsing script throws an exception, it returns -1. This could potentially cancel a run failure and result in a false negative.
+    # Absolute value is taken to avoid that.
+    sys.exit(abs(total_num_func_failures) + abs(total_num_qor_failures))
 
 
 def display_qor(reg_test):

vtr_flow/scripts/python_libs/vtr/parse_vtr_task.py

@@ -422,14 +422,6 @@ def check_two_files(
     for (arch, circuit, script_params), _ in first_results.all_metrics().items():
         first_primary_keys.append((arch, circuit, script_params))
 
-    # Ensure that first result file  has all the second result file cases
-    for arch, circuit, script_params in second_primary_keys:
-        if first_results.metrics(arch, circuit, script_params) is None:
-            raise InspectError(
-                "Required case {}/{} missing from {} results: {}".format(
-                    arch, circuit, first_name, first_results_filepath
-                )
-            )
 
     # Warn about any elements in first result file that are not found in second result file
     for arch, circuit, script_params in first_primary_keys:
@@ -444,6 +436,14 @@ def check_two_files(
     for arch, circuit, script_params in second_primary_keys:
         second_metrics = second_results.metrics(arch, circuit, script_params)
         first_metrics = first_results.metrics(arch, circuit, script_params)
+
+        if first_metrics == None:
+            num_qor_failures += 1
+            print("Required case {}/{} missing from {} results: {}".format(
+                    arch, circuit, first_name, first_results_filepath
+                ))
+            continue
+        
         first_fail = True
         for metric in pass_requirements.keys():