Merge pull request #8 from unity/AddKnownIssue

amaestas13 · GitHub Enterprise · commit 395d9199a7c7 · 2023-01-04T13:11:52.000-08:00
Add known issue Support
diff --git a/UnityPerformanceBenchmarkReporter/Entities/Data.cs b/UnityPerformanceBenchmarkReporter/Entities/Data.cs
@@ -26,6 +26,8 @@ public class SampleGroup
         public double Average;
         public double StandardDeviation;
         public double Sum;
+        public bool ContainsKnownIssue;
+        public string KnownIssueDetails = "";
 
         public SampleGroup(string name, SampleUnit unit, bool increaseIsBetter)
         {
diff --git a/UnityPerformanceBenchmarkReporter/Entities/SampleGroup.cs b/UnityPerformanceBenchmarkReporter/Entities/SampleGroup.cs
@@ -30,6 +30,8 @@ public class SampleGroupDefinition
         public double Percentile;
 
         public bool FailOnBaseline;
+        public bool ContainsKnownIssue;
+        public string KnownIssueDetails = "";
     }
 
     public enum AggregationType
diff --git a/UnityPerformanceBenchmarkReporter/Entities/SampleGroupResult.cs b/UnityPerformanceBenchmarkReporter/Entities/SampleGroupResult.cs
@@ -13,6 +13,9 @@ public class SampleGroupResult
         public double Percentile;
         public bool Regressed;
         public bool Progressed;
+        public bool RegressedKnown;
+        public bool ContainsKnownIssue;
+        public string KnownIssueDetails;
         public double Min;
         public double Max;
         public double Median;
diff --git a/UnityPerformanceBenchmarkReporter/PerformanceTestRunProcessor.cs b/UnityPerformanceBenchmarkReporter/PerformanceTestRunProcessor.cs
@@ -9,7 +9,8 @@ internal enum MeasurementResult
     {
         Neutral = 0,
         Regression = 1,
-        Progression = 2
+        Progression = 2,
+        RegressionKnown = 3
     }
 
     public class PerformanceTestRunProcessor
@@ -45,6 +46,8 @@ public List<TestResult> GetTestResults(
                         Average = sampleGroup.Average,
                         StandardDeviation = sampleGroup.StandardDeviation,
                         PercentileValue = sampleGroup.PercentileValue,
+                        ContainsKnownIssue = sampleGroup.Definition.ContainsKnownIssue,
+                        KnownIssueDetails = sampleGroup.Definition.KnownIssueDetails,
                         Sum = sampleGroup.Sum,
                         Zeroes = sampleGroup.Zeroes,
                         SampleCount = sampleGroup.SampleCount,
@@ -89,16 +92,22 @@ public void UpdateTestResultsBasedOnBaselineResults(List<TestResult> baselineTes
                         {
                             sampleGroupResult.Regressed = true;
                             sampleGroupResult.Progressed = false;
+                            sampleGroupResult.RegressedKnown = false;
                         }
                         else if (res == MeasurementResult.Progression)
                         {
                             sampleGroupResult.Regressed = false;
                             sampleGroupResult.Progressed = true;
+                            sampleGroupResult.RegressedKnown = false;
+                        }else if(res == MeasurementResult.RegressionKnown){
+                            sampleGroupResult.Regressed = true;
+                            sampleGroupResult.Progressed = false;
+                            sampleGroupResult.RegressedKnown = true;
                         }
                     }
                 }
 
-                if (testResult.SampleGroupResults.Any(r => r.Regressed))
+                if (testResult.SampleGroupResults.Any(r => r.Regressed && r.RegressedKnown == false))
                 {
                     testResult.State = (int)TestState.Failure;
                 }
@@ -173,6 +182,9 @@ private MeasurementResult DeterminePerformanceResult(SampleGroupResult sampleGro
                 if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) < negativeThresholdValue.TruncToSigFig(sigFig))
                 {
                     measurementResult = MeasurementResult.Regression;
+                    
+                    if(sampleGroup.ContainsKnownIssue)
+                        measurementResult = MeasurementResult.RegressionKnown;
                 }
                 if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) > positiveThresholdValue.TruncToSigFig(sigFig))
                 {
@@ -184,6 +196,9 @@ private MeasurementResult DeterminePerformanceResult(SampleGroupResult sampleGro
                 if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) > positiveThresholdValue.TruncToSigFig(sigFig))
                 {
                     measurementResult = MeasurementResult.Regression;
+
+                    if(sampleGroup.ContainsKnownIssue)
+                        measurementResult = MeasurementResult.RegressionKnown;
                 }
                 if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) < negativeThresholdValue.TruncToSigFig(sigFig))
                 {
diff --git a/UnityPerformanceBenchmarkReporter/Program.cs b/UnityPerformanceBenchmarkReporter/Program.cs
@@ -117,6 +117,7 @@ private static int Main(string[] args)
                 performanceBenchmark.ReportDirPath,
                 performanceBenchmark.BaselineResultFilesExist);
             WriteProgressedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
+            WriteRegressedKnownTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
             int result = WriteFailedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
             WriteLine($"Finished with Result {result}");
             return result;
@@ -128,7 +129,7 @@ private static int WriteFailedTestsAndMetricsToConsole(PerformanceTestRunResult[
                 .Any(tr => tr.State == (int)TestState.Failure);
             if (failedTestsExist)
             {
-                WriteLine("FAILURE: One ore more performance test metric aggregations is out of threshold from the baseline value.");
+                WriteLine("FAILURE: One ore more performance test metric aggregations is out of threshold from the baseline value. REGRESSIONS!");
                 WriteLine("-------------------------------------");
                 WriteLine(" Performance tests with failed metrics");
                 WriteLine("-------------------------------------");
@@ -162,6 +163,57 @@ private static int WriteFailedTestsAndMetricsToConsole(PerformanceTestRunResult[
             return performanceBenchmark.FailOnBaseline && failedTestsExist ? 1 : 0;
         }
 
+        private static void WriteRegressedKnownTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
+        {
+            bool loggedHeader = false;
+            var passedTestsExist = performanceTestResults.SelectMany(ptr => ptr.TestResults)
+                .Any(tr => tr.State == (int)TestState.Success);
+            if (passedTestsExist)
+            {
+
+                foreach (var performanceTestRunResult in performanceTestResults)
+                {
+                    var passedTests = performanceTestRunResult.TestResults.Where(tr => tr.State == (int)TestState.Success);
+                    if (passedTests.Any())
+                    {
+                        foreach (var tests in passedTests)
+                        {
+                            if (tests.SampleGroupResults.Any(sgr => sgr.RegressedKnown && sgr.Regressed))
+                            {
+                                if (!loggedHeader)
+                                {
+                                    loggedHeader = true;
+                                    WriteLine("Info: One ore more performance test metric aggregations is out of threshold from the baseline value. KNOWN REGRESSIONS!");
+                                    WriteLine("-------------------------------------");
+                                    WriteLine(" Performance tests with Known Regressions metrics");
+                                    WriteLine("-------------------------------------");
+                                }
+
+                                ++indentLevel;
+                                WriteLine("{0}", tests.TestName);
+
+                                var progressedSgs = tests.SampleGroupResults.Where(sgr => sgr.Progressed);
+                                foreach (var sampleGroupResult in progressedSgs)
+                                {
+                                    WriteLine("----");
+                                    WriteLine("Metric        : {0}", sampleGroupResult.SampleGroupName);
+                                    WriteLine("Aggregation   : {0}", sampleGroupResult.AggregationType);
+                                    WriteLine("New Value  : {0,8:F2}", sampleGroupResult.AggregatedValue);
+                                    WriteLine("Baseline Value: {0,8:F2}", sampleGroupResult.BaselineValue);
+                                    WriteLine("Threshold %   : {0,8:F2}", sampleGroupResult.Threshold);
+                                    WriteLine("Actual Diff % : {0,8:F2}", Math.Abs(sampleGroupResult.BaselineValue - sampleGroupResult.AggregatedValue) / sampleGroupResult.BaselineValue);
+                                    WriteLine($"Known Issue: {sampleGroupResult.KnownIssueDetails}");
+                                }
+                                --indentLevel;
+                                WriteLine("\r\n");
+                            }
+                        }
+                    }
+                }
+            }
+
+        }
+
         private static void WriteProgressedTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
         {
             bool loggedHeader = false;
@@ -182,7 +234,7 @@ private static void WriteProgressedTestsAndMetricsToConsole(PerformanceTestRunRe
                                 if (!loggedHeader)
                                 {
                                     loggedHeader = true;
-                                    WriteLine("Info: One ore more performance test metric aggregations is out of threshold from the baseline value.");
+                                    WriteLine("Info: One ore more performance test metric aggregations is out of threshold from the baseline value. PROGRESSIONS!");
                                     WriteLine("-------------------------------------");
                                     WriteLine(" Performance tests with Progressed metrics");
                                     WriteLine("-------------------------------------");
diff --git a/UnityPerformanceBenchmarkReporter/TestResultJsonParser.cs b/UnityPerformanceBenchmarkReporter/TestResultJsonParser.cs
@@ -166,7 +166,9 @@ private static PerformanceTestRun ParseJsonV2(string json)
                                 Name = sg.Name,
                                 SampleUnit = (Entities.SampleUnit)sg.Unit,
                                 IncreaseIsBetter = sg.IncreaseIsBetter,
-                                Threshold = sg.Threshold
+                                Threshold = sg.Threshold,
+                                ContainsKnownIssue = sg.ContainsKnownIssue,
+                                KnownIssueDetails = sg.KnownIssueDetails
                             }
                         }).ToList()
                     };
diff --git a/UnityPerformanceBenchmarkReporter/TestResultXmlParser.cs b/UnityPerformanceBenchmarkReporter/TestResultXmlParser.cs
@@ -96,7 +96,9 @@ private static void DeserializeTestResultsV2(IEnumerable<XElement> output, Perfo
                                     Name = sg.Name,
                                     SampleUnit = (Entities.SampleUnit)sg.Unit,
                                     IncreaseIsBetter = sg.IncreaseIsBetter,
-                                    Threshold = sg.Threshold
+                                    Threshold = sg.Threshold,
+                                    ContainsKnownIssue = sg.ContainsKnownIssue,
+                                    KnownIssueDetails = sg.KnownIssueDetails
                                 }
                             }).ToList()
                         };

Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,8 @@ public class SampleGroup`
`26`	`26`	`public double Average;`
`27`	`27`	`public double StandardDeviation;`
`28`	`28`	`public double Sum;`
	`29`	`+ public bool ContainsKnownIssue;`
	`30`	`+ public string KnownIssueDetails = "";`
`29`	`31`
`30`	`32`	`public SampleGroup(string name, SampleUnit unit, bool increaseIsBetter)`
`31`	`33`	`{`
Original file line number	Diff line number	Diff line change
`@@ -30,6 +30,8 @@ public class SampleGroupDefinition`
`30`	`30`	`public double Percentile;`
`31`	`31`
`32`	`32`	`public bool FailOnBaseline;`
	`33`	`+ public bool ContainsKnownIssue;`
	`34`	`+ public string KnownIssueDetails = "";`
`33`	`35`	`}`
`34`	`36`
`35`	`37`	`public enum AggregationType`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,8 @@ internal enum MeasurementResult`
`9`	`9`	`{`
`10`	`10`	`Neutral = 0,`
`11`	`11`	`Regression = 1,`
`12`		`- Progression = 2`
	`12`	`+ Progression = 2,`
	`13`	`+ RegressionKnown = 3`
`13`	`14`	`}`
`14`	`15`
`15`	`16`	`public class PerformanceTestRunProcessor`
`@@ -45,6 +46,8 @@ public List<TestResult> GetTestResults(`
`45`	`46`	`Average = sampleGroup.Average,`
`46`	`47`	`StandardDeviation = sampleGroup.StandardDeviation,`
`47`	`48`	`PercentileValue = sampleGroup.PercentileValue,`
	`49`	`+ ContainsKnownIssue = sampleGroup.Definition.ContainsKnownIssue,`
	`50`	`+ KnownIssueDetails = sampleGroup.Definition.KnownIssueDetails,`
`48`	`51`	`Sum = sampleGroup.Sum,`
`49`	`52`	`Zeroes = sampleGroup.Zeroes,`
`50`	`53`	`SampleCount = sampleGroup.SampleCount,`
`@@ -89,16 +92,22 @@ public void UpdateTestResultsBasedOnBaselineResults(List<TestResult> baselineTes`
`89`	`92`	`{`
`90`	`93`	`sampleGroupResult.Regressed = true;`
`91`	`94`	`sampleGroupResult.Progressed = false;`
	`95`	`+ sampleGroupResult.RegressedKnown = false;`
`92`	`96`	`}`
`93`	`97`	`else if (res == MeasurementResult.Progression)`
`94`	`98`	`{`
`95`	`99`	`sampleGroupResult.Regressed = false;`
`96`	`100`	`sampleGroupResult.Progressed = true;`
	`101`	`+ sampleGroupResult.RegressedKnown = false;`
	`102`	`+ }else if(res == MeasurementResult.RegressionKnown){`
	`103`	`+ sampleGroupResult.Regressed = true;`
	`104`	`+ sampleGroupResult.Progressed = false;`
	`105`	`+ sampleGroupResult.RegressedKnown = true;`
`97`	`106`	`}`
`98`	`107`	`}`
`99`	`108`	`}`
`100`	`109`
`101`		`- if (testResult.SampleGroupResults.Any(r => r.Regressed))`
	`110`	`+ if (testResult.SampleGroupResults.Any(r => r.Regressed && r.RegressedKnown == false))`
`102`	`111`	`{`
`103`	`112`	`testResult.State = (int)TestState.Failure;`
`104`	`113`	`}`
`@@ -173,6 +182,9 @@ private MeasurementResult DeterminePerformanceResult(SampleGroupResult sampleGro`
`173`	`182`	`if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) < negativeThresholdValue.TruncToSigFig(sigFig))`
`174`	`183`	`{`
`175`	`184`	`measurementResult = MeasurementResult.Regression;`
	`185`	`+`
	`186`	`+ if(sampleGroup.ContainsKnownIssue)`
	`187`	`+ measurementResult = MeasurementResult.RegressionKnown;`
`176`	`188`	`}`
`177`	`189`	`if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) > positiveThresholdValue.TruncToSigFig(sigFig))`
`178`	`190`	`{`
`@@ -184,6 +196,9 @@ private MeasurementResult DeterminePerformanceResult(SampleGroupResult sampleGro`
`184`	`196`	`if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) > positiveThresholdValue.TruncToSigFig(sigFig))`
`185`	`197`	`{`
`186`	`198`	`measurementResult = MeasurementResult.Regression;`
	`199`	`+`
	`200`	`+ if(sampleGroup.ContainsKnownIssue)`
	`201`	`+ measurementResult = MeasurementResult.RegressionKnown;`
`187`	`202`	`}`
`188`	`203`	`if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) < negativeThresholdValue.TruncToSigFig(sigFig))`
`189`	`204`	`{`