Skip to content

Commit 22ec1d2

Browse files
amaestas13amaestas13
committed
First Implementation
1 parent c7a4c24 commit 22ec1d2

File tree

7 files changed

+78
-5
lines changed

7 files changed

+78
-5
lines changed

‎UnityPerformanceBenchmarkReporter/Entities/Data.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ public class SampleGroup
2626
public double Average;
2727
public double StandardDeviation;
2828
public double Sum;
29+
public bool ContainsKnownIssue;
30+
public string KnownIssueDetails = "";
2931

3032
public SampleGroup(string name, SampleUnit unit, bool increaseIsBetter)
3133
{

‎UnityPerformanceBenchmarkReporter/Entities/SampleGroup.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ public class SampleGroupDefinition
3030
public double Percentile;
3131

3232
public bool FailOnBaseline;
33+
public bool ContainsKnownIssue;
34+
public string KnownIssueDetails = "";
3335
}
3436

3537
public enum AggregationType

‎UnityPerformanceBenchmarkReporter/Entities/SampleGroupResult.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public class SampleGroupResult
1313
public double Percentile;
1414
public bool Regressed;
1515
public bool Progressed;
16+
public bool RegressedKnown;
1617
public double Min;
1718
public double Max;
1819
public double Median;

‎UnityPerformanceBenchmarkReporter/PerformanceTestRunProcessor.cs‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ internal enum MeasurementResult
99
{
1010
Neutral = 0,
1111
Regression = 1,
12-
Progression = 2
12+
Progression = 2,
13+
RegressionKnown = 3
1314
}
1415

1516
public class PerformanceTestRunProcessor
@@ -89,16 +90,22 @@ public void UpdateTestResultsBasedOnBaselineResults(List<TestResult> baselineTes
8990
{
9091
sampleGroupResult.Regressed = true;
9192
sampleGroupResult.Progressed = false;
93+
sampleGroupResult.RegressedKnown = false;
9294
}
9395
else if (res == MeasurementResult.Progression)
9496
{
9597
sampleGroupResult.Regressed = false;
9698
sampleGroupResult.Progressed = true;
99+
sampleGroupResult.RegressedKnown = false;
100+
}else if(res = MeasurementResult.RegressionKnown){
101+
sampleGroupResult.Regressed = true;
102+
sampleGroupResult.Progressed = false;
103+
sampleGroupResult.RegressedKnown = true;
97104
}
98105
}
99106
}
100107

101-
if (testResult.SampleGroupResults.Any(r => r.Regressed))
108+
if (testResult.SampleGroupResults.Any(r => r.Regressed && r.RegressedKnown == false))
102109
{
103110
testResult.State = (int)TestState.Failure;
104111
}
@@ -173,6 +180,9 @@ private MeasurementResult DeterminePerformanceResult(SampleGroupResult sampleGro
173180
if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) < negativeThresholdValue.TruncToSigFig(sigFig))
174181
{
175182
measurementResult = MeasurementResult.Regression;
183+
184+
if(sampleGroup.ContainsKnownIssue)
185+
measurementResult = MeasurementResult.RegressionKnown;
176186
}
177187
if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) > positiveThresholdValue.TruncToSigFig(sigFig))
178188
{
@@ -184,6 +194,9 @@ private MeasurementResult DeterminePerformanceResult(SampleGroupResult sampleGro
184194
if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) > positiveThresholdValue.TruncToSigFig(sigFig))
185195
{
186196
measurementResult = MeasurementResult.Regression;
197+
198+
if(sampleGroup.ContainsKnownIssue)
199+
measurementResult = MeasurementResult.RegressionKnown;
187200
}
188201
if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) < negativeThresholdValue.TruncToSigFig(sigFig))
189202
{

‎UnityPerformanceBenchmarkReporter/Program.cs‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ private static int Main(string[] args)
117117
performanceBenchmark.ReportDirPath,
118118
performanceBenchmark.BaselineResultFilesExist);
119119
WriteProgressedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
120+
WriteRegressedKnownTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
120121
int result = WriteFailedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
121122
WriteLine($"Finished with Result {result}");
122123
return result;
@@ -128,7 +129,7 @@ private static int WriteFailedTestsAndMetricsToConsole(PerformanceTestRunResult[
128129
.Any(tr => tr.State == (int)TestState.Failure);
129130
if (failedTestsExist)
130131
{
131-
WriteLine("FAILURE: One ore more performance test metric aggregations is out of threshold from the baseline value.");
132+
WriteLine("FAILURE: One ore more performance test metric aggregations is out of threshold from the baseline value. REGRESSIONS!");
132133
WriteLine("-------------------------------------");
133134
WriteLine(" Performance tests with failed metrics");
134135
WriteLine("-------------------------------------");
@@ -162,6 +163,56 @@ private static int WriteFailedTestsAndMetricsToConsole(PerformanceTestRunResult[
162163
return performanceBenchmark.FailOnBaseline && failedTestsExist ? 1 : 0;
163164
}
164165

166+
private static void WriteRegressedKnownTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
167+
{
168+
bool loggedHeader = false;
169+
var passedTestsExist = performanceTestResults.SelectMany(ptr => ptr.TestResults)
170+
.Any(tr => tr.State == (int)TestState.Success);
171+
if (passedTestsExist)
172+
{
173+
174+
foreach (var performanceTestRunResult in performanceTestResults)
175+
{
176+
var passedTests = performanceTestRunResult.TestResults.Where(tr => tr.State == (int)TestState.Success);
177+
if (passedTests.Any())
178+
{
179+
foreach (var tests in passedTests)
180+
{
181+
if (tests.SampleGroupResults.Any(sgr => sgr.RegressedKnown && sgr.Regressed))
182+
{
183+
if (!loggedHeader)
184+
{
185+
loggedHeader = true;
186+
WriteLine("Info: One ore more performance test metric aggregations is out of threshold from the baseline value. KNOWN REGRESSIONS!");
187+
WriteLine("-------------------------------------");
188+
WriteLine(" Performance tests with Known Regressions metrics");
189+
WriteLine("-------------------------------------");
190+
}
191+
192+
++indentLevel;
193+
WriteLine("{0}", tests.TestName);
194+
195+
var progressedSgs = tests.SampleGroupResults.Where(sgr => sgr.Progressed);
196+
foreach (var sampleGroupResult in progressedSgs)
197+
{
198+
WriteLine("----");
199+
WriteLine("Metric : {0}", sampleGroupResult.SampleGroupName);
200+
WriteLine("Aggregation : {0}", sampleGroupResult.AggregationType);
201+
WriteLine("New Value : {0,8:F2}", sampleGroupResult.AggregatedValue);
202+
WriteLine("Baseline Value: {0,8:F2}", sampleGroupResult.BaselineValue);
203+
WriteLine("Threshold % : {0,8:F2}", sampleGroupResult.Threshold);
204+
WriteLine("Actual Diff % : {0,8:F2}", Math.Abs(sampleGroupResult.BaselineValue - sampleGroupResult.AggregatedValue) / sampleGroupResult.BaselineValue);
205+
}
206+
--indentLevel;
207+
WriteLine("\r\n");
208+
}
209+
}
210+
}
211+
}
212+
}
213+
214+
}
215+
165216
private static void WriteProgressedTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
166217
{
167218
bool loggedHeader = false;
@@ -182,7 +233,7 @@ private static void WriteProgressedTestsAndMetricsToConsole(PerformanceTestRunRe
182233
if (!loggedHeader)
183234
{
184235
loggedHeader = true;
185-
WriteLine("Info: One ore more performance test metric aggregations is out of threshold from the baseline value.");
236+
WriteLine("Info: One ore more performance test metric aggregations is out of threshold from the baseline value. PROGRESSIONS!");
186237
WriteLine("-------------------------------------");
187238
WriteLine(" Performance tests with Progressed metrics");
188239
WriteLine("-------------------------------------");

‎UnityPerformanceBenchmarkReporter/TestResultJsonParser.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,9 @@ private static PerformanceTestRun ParseJsonV2(string json)
166166
Name = sg.Name,
167167
SampleUnit = (Entities.SampleUnit)sg.Unit,
168168
IncreaseIsBetter = sg.IncreaseIsBetter,
169-
Threshold = sg.Threshold
169+
Threshold = sg.Threshold,
170+
ContainsKnownIssue = sg.ContainsKnownIssue,
171+
KnownIssueDetails = sg.KnownIssueDetails
170172
}
171173
}).ToList()
172174
};

‎UnityPerformanceBenchmarkReporter/TestResultXmlParser.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@ private static void DeserializeTestResultsV2(IEnumerable<XElement> output, Perfo
9797
SampleUnit = (Entities.SampleUnit)sg.Unit,
9898
IncreaseIsBetter = sg.IncreaseIsBetter,
9999
Threshold = sg.Threshold
100+
ContainsKnownIssue = sg.ContainsKnownIssue,
101+
KnownIssueDetails = sg.KnownIssueDetails
100102
}
101103
}).ToList()
102104
};

0 commit comments

Comments
 (0)