Skip to content

Commit 8f28284

Browse files
amaestas13amaestas13
committed
Added Logging for Progresed Tests
Fixed Tests on Json side Added Stndard deveation to threshold calculation
1 parent 572aee0 commit 8f28284

File tree

5 files changed

+114
-49
lines changed

5 files changed

+114
-49
lines changed

‎UnityPerformanceBenchmarkReporter/Entities/SampleGroupResult.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ public class SampleGroupResult
1212
public string AggregationType;
1313
public double Percentile;
1414
public bool Regressed;
15+
public bool Progressed;
1516
public double Min;
1617
public double Max;
1718
public double Median;

‎UnityPerformanceBenchmarkReporter/PerformanceTestRunProcessor.cs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public List<TestResult> GetTestResults(
2626
TestName = testName,
2727
TestCategories = performanceTestRun.Results.First(r => r.TestName == testName).TestCategories,
2828
TestVersion = performanceTestRun.Results.First(r => r.TestName == testName).TestVersion,
29-
State = (int) TestState.Success,
29+
State = (int)TestState.Success,
3030
SampleGroupResults = new List<SampleGroupResult>()
3131
};
3232
foreach (var sampleGroup in mergedTestExecutions[testName])
@@ -83,13 +83,24 @@ public void UpdateTestResultsBasedOnBaselineResults(List<TestResult> baselineTes
8383
sampleGroupResult.BaselineValue = baselineSampleGroupResult.AggregatedValue;
8484
sampleGroupResult.Threshold = baselineSampleGroupResult.Threshold;
8585

86-
sampleGroupResult.Regressed = DeterminePerformanceResult(sampleGroupResult, sigfig) == MeasurementResult.Regression;
86+
var res = DeterminePerformanceResult(sampleGroupResult, sigfig);
87+
88+
if (res == MeasurementResult.Regression)
89+
{
90+
sampleGroupResult.Regressed = true;
91+
sampleGroupResult.Progressed = false;
92+
}
93+
else if (res == MeasurementResult.Progression)
94+
{
95+
sampleGroupResult.Regressed = false;
96+
sampleGroupResult.Progressed = true;
97+
}
8798
}
8899
}
89100

90101
if (testResult.SampleGroupResults.Any(r => r.Regressed))
91102
{
92-
testResult.State = (int) TestState.Failure;
103+
testResult.State = (int)TestState.Failure;
93104
}
94105
}
95106
}
@@ -154,6 +165,9 @@ private MeasurementResult DeterminePerformanceResult(SampleGroupResult sampleGro
154165
var measurementResult = MeasurementResult.Neutral;
155166
var positiveThresholdValue = sampleGroup.BaselineValue + sampleGroup.BaselineValue * sampleGroup.Threshold;
156167
var negativeThresholdValue = sampleGroup.BaselineValue - sampleGroup.BaselineValue * sampleGroup.Threshold;
168+
positiveThresholdValue += sampleGroup.StandardDeviation;
169+
negativeThresholdValue -= sampleGroup.StandardDeviation;
170+
157171
if (sampleGroup.IncreaseIsBetter)
158172
{
159173
if (sampleGroup.AggregatedValue.TruncToSigFig(sigFig) < negativeThresholdValue.TruncToSigFig(sigFig))

‎UnityPerformanceBenchmarkReporter/Program.cs‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ private static int Main(string[] args)
116116
performanceBenchmark.SigFig,
117117
performanceBenchmark.ReportDirPath,
118118
performanceBenchmark.BaselineResultFilesExist);
119-
119+
WriteProgressedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
120120
int result = WriteFailedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark);
121121
WriteLine($"Finished with Result {result}");
122122
return result;
@@ -162,6 +162,56 @@ private static int WriteFailedTestsAndMetricsToConsole(PerformanceTestRunResult[
162162
return performanceBenchmark.FailOnBaseline && failedTestsExist ? 1 : 0;
163163
}
164164

165+
private static void WriteProgressedTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
166+
{
167+
bool loggedHeader = false;
168+
var passedTestsExist = performanceTestResults.SelectMany(ptr => ptr.TestResults)
169+
.Any(tr => tr.State == (int)TestState.Success);
170+
if (passedTestsExist)
171+
{
172+
173+
foreach (var performanceTestRunResult in performanceTestResults)
174+
{
175+
var passedTests = performanceTestRunResult.TestResults.Where(tr => tr.State == (int)TestState.Success);
176+
if (passedTests.Any())
177+
{
178+
foreach (var tests in passedTests)
179+
{
180+
if (tests.SampleGroupResults.Any(sgr => sgr.Progressed))
181+
{
182+
if (!loggedHeader)
183+
{
184+
loggedHeader = true;
185+
WriteLine("Info: One ore more performance test metric aggregations is out of threshold from the baseline value.");
186+
WriteLine("-------------------------------------");
187+
WriteLine(" Performance tests with Progressed metrics");
188+
WriteLine("-------------------------------------");
189+
}
190+
191+
++indentLevel;
192+
WriteLine("{0}", tests.TestName);
193+
194+
var progressedSgs = tests.SampleGroupResults.Where(sgr => sgr.Progressed);
195+
foreach (var sampleGroupResult in progressedSgs)
196+
{
197+
WriteLine("----");
198+
WriteLine("Metric : {0}", sampleGroupResult.SampleGroupName);
199+
WriteLine("Aggregation : {0}", sampleGroupResult.AggregationType);
200+
WriteLine("New Value : {0,8:F2}", sampleGroupResult.AggregatedValue);
201+
WriteLine("Baseline Value: {0,8:F2}", sampleGroupResult.BaselineValue);
202+
WriteLine("Threshold % : {0,8:F2}", sampleGroupResult.Threshold);
203+
WriteLine("Actual Diff % : {0,8:F2}", Math.Abs(sampleGroupResult.BaselineValue - sampleGroupResult.AggregatedValue) / sampleGroupResult.BaselineValue);
204+
}
205+
--indentLevel;
206+
WriteLine("\r\n");
207+
}
208+
}
209+
}
210+
}
211+
}
212+
213+
}
214+
165215
private static void WriteLine(string format, params object[] args)
166216
{
167217
Console.Write(new string('\t', indentLevel));

‎UnityPerformanceBenchmarkReporter/TestResultJsonParser.cs‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -145,29 +145,29 @@ private static PerformanceTestRun ParseJsonV2(string json)
145145
foreach (var res in run.Results)
146146
{
147147
var pt = new PerformanceTestResult()
148+
{
149+
TestCategories = res.Categories,
150+
TestName = res.Name,
151+
TestVersion = res.Version,
152+
SampleGroups = res.SampleGroups.Select(sg => new Entities.SampleGroup
148153
{
149-
TestCategories = res.Categories,
150-
TestName = res.Name,
151-
TestVersion = res.Version,
152-
SampleGroups = res.SampleGroups.Select(sg => new Entities.SampleGroup
154+
Samples = sg.Samples,
155+
Average = sg.Average,
156+
Max = sg.Max,
157+
Median = sg.Median,
158+
Min = sg.Min,
159+
Sum = sg.Sum,
160+
StandardDeviation = sg.StandardDeviation,
161+
SampleCount = sg.Samples.Count,
162+
Definition = new SampleGroupDefinition()
153163
{
154-
Samples = sg.Samples,
155-
Average = sg.Average,
156-
Max = sg.Max,
157-
Median = sg.Median,
158-
Min = sg.Min,
159-
Sum = sg.Sum,
160-
StandardDeviation = sg.StandardDeviation,
161-
SampleCount = sg.Samples.Count,
162-
Definition = new SampleGroupDefinition()
163-
{
164-
Name = sg.Name,
165-
SampleUnit = (Entities.SampleUnit)sg.Unit,
166-
IncreaseIsBetter = sg.IncreaseIsBetter
167-
}
168-
}).ToList()
169-
};
170-
testRun.Results.Add(pt);
164+
Name = sg.Name,
165+
SampleUnit = (Entities.SampleUnit)sg.Unit,
166+
IncreaseIsBetter = sg.IncreaseIsBetter
167+
}
168+
}).ToList()
169+
};
170+
testRun.Results.Add(pt);
171171
}
172172

173173
return testRun;

‎UnityPerformanceBenchmarkReporterTests/PerformanceBenchmarkTestsJson.cs‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ public void Setup()
3232
public void VerifyV1_AddPerformanceTestRunResults()
3333
{
3434
// Arrange
35-
var resultJsonFilePath = EnsureFullPath("results.Json");
36-
var args = new[] { "--format=Json",string.Format("--testresultsxmlsource={0}", resultJsonFilePath), "--version=1" };
35+
var resultJsonFilePath = EnsureFullPath("results.json");
36+
var args = new[] { "--format=json", string.Format("--testresultsxmlsource={0}", resultJsonFilePath), "--version=1" };
3737
optionsParser.ParseOptions(PerformanceBenchmark, args);
3838

3939
// Act
@@ -52,11 +52,11 @@ public void VerifyV1_AddPerformanceTestRunResults()
5252
public void VerifyV1_AddPerformanceTestRunResults_TwoResultFiles()
5353
{
5454
// Arrange
55-
var resultJsonFilePath = EnsureFullPath("results.Json");
56-
var resultFileName2 = EnsureFullPath("results2.Json");
55+
var resultJsonFilePath = EnsureFullPath("results.json");
56+
var resultFileName2 = EnsureFullPath("results.json");
5757
var args = new[]
5858
{
59-
"--format=Json",
59+
"--format=json",
6060
string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
6161
string.Format("--testresultsxmlsource={0}", resultFileName2)
6262
, "--version=1"
@@ -79,11 +79,11 @@ public void VerifyV1_AddPerformanceTestRunResults_TwoResultFiles()
7979
public void VerifyV1_AddPerformanceTestRunResults_OneResultFiles_OneResultDirectory()
8080
{
8181
// Arrange
82-
var resultJsonFilePath = EnsureFullPath("results.Json");
82+
var resultJsonFilePath = EnsureFullPath("results.json");
8383
var resultsJsonDir = EnsureFullPath("ResultsJson");
8484
var args = new[]
8585
{
86-
"--format=Json",
86+
"--format=json",
8787
string.Format("--testresultsxmlsource={0}", resultJsonFilePath),
8888
string.Format("--testresultsxmlsource={0}", resultsJsonDir)
8989
, "--version=1"
@@ -107,8 +107,8 @@ public void VerifyV1_AddPerformanceTestRunResults_OneResultFiles_OneResultDirect
107107
public void VerifyV1_AddBaselinePerformanceTestRunResults()
108108
{
109109
// Arrange
110-
var resultJsonFilePath = EnsureFullPath("results.Json");
111-
var baselineJsonFilePath = EnsureFullPath("baseline.Json");
110+
var resultJsonFilePath = EnsureFullPath("results.json");
111+
var baselineJsonFilePath = EnsureFullPath("baseline.json");
112112
var args = new[]
113113
{
114114
"--format=Json",
@@ -161,8 +161,8 @@ public void VerifyV1_AddBaselinePerformanceTestRunResultsDirectory()
161161
public void VerifyV1_Verify_AddBaselineAndNonBaselinePerformanceTestRunResults()
162162
{
163163
// Arrange
164-
var resultJsonFilePath = EnsureFullPath("results.Json");
165-
var baselineJsonFilePath = EnsureFullPath("baseline.Json");
164+
var resultJsonFilePath = EnsureFullPath("results.json");
165+
var baselineJsonFilePath = EnsureFullPath("baseline.json");
166166
var args = new[]
167167
{
168168
"--format=Json",
@@ -192,11 +192,11 @@ public void VerifyV1_Verify_AddBaselineAndNonBaselinePerformanceTestRunResults()
192192
Assert.IsTrue(baselinePerformanceTestRunResults.Count > 0);
193193
}
194194

195-
public void VerifyV2_AddPerformanceTestRunResults()
195+
public void VerifyV2_AddPerformanceTestRunResults()
196196
{
197197
// Arrange
198-
var resultJsonFilePath = EnsureFullPath("baseline2.Json");
199-
var args = new[] { string.Format("--format=Json","--testresultsxmlsource={0}", resultJsonFilePath), "--version=2" };
198+
var resultJsonFilePath = EnsureFullPath("baseline2.json");
199+
var args = new[] { string.Format("--format=Json", "--testresultsxmlsource={0}", resultJsonFilePath), "--version=2" };
200200
optionsParser.ParseOptions(PerformanceBenchmark, args);
201201

202202
// Act
@@ -215,8 +215,8 @@ public void VerifyV2_AddPerformanceTestRunResults()
215215
public void VerifyV2_AddPerformanceTestRunResults_TwoResultFiles()
216216
{
217217
// Arrange
218-
var resultJsonFilePath = EnsureFullPath("ResultsJson2/results2.Json");
219-
var resultFileName2 = EnsureFullPath("results2.Json");
218+
var resultJsonFilePath = EnsureFullPath("ResultsJson2/results2.json");
219+
var resultFileName2 = EnsureFullPath("results2.json");
220220
var args = new[]
221221
{
222222
"--format=Json",
@@ -242,7 +242,7 @@ public void VerifyV2_AddPerformanceTestRunResults_TwoResultFiles()
242242
public void VerifyV2_AddPerformanceTestRunResults_OneResultFiles_OneResultDirectory()
243243
{
244244
// Arrange
245-
var resultJsonFilePath = EnsureFullPath("results2.Json");
245+
var resultJsonFilePath = EnsureFullPath("results2.json");
246246
var resultsJsonDir = EnsureFullPath("ResultsJson2");
247247
var args = new[]
248248
{
@@ -270,8 +270,8 @@ public void VerifyV2_AddPerformanceTestRunResults_OneResultFiles_OneResultDirect
270270
public void VerifyV2_AddBaselinePerformanceTestRunResults()
271271
{
272272
// Arrange
273-
var resultJsonFilePath = EnsureFullPath("results2.Json");
274-
var baselineJsonFilePath = EnsureFullPath("baseline2.Json");
273+
var resultJsonFilePath = EnsureFullPath("results2.json");
274+
var baselineJsonFilePath = EnsureFullPath("baseline2.json");
275275
var args = new[]
276276
{
277277
"--format=Json",
@@ -298,8 +298,8 @@ public void VerifyV2_AddBaselinePerformanceTestRunResults()
298298
public void VerifyV2_AddBaselinePerformanceTestRunResultsDirectory()
299299
{
300300
// Arrange
301-
var resultJsonFilePath = EnsureFullPath("results2.Json");
302-
var baselineJsonFilePath = EnsureFullPath("baseline2.Json");
301+
var resultJsonFilePath = EnsureFullPath("results2.json");
302+
var baselineJsonFilePath = EnsureFullPath("baseline2.json");
303303
var args = new[]
304304
{
305305
"--format=Json",
@@ -324,8 +324,8 @@ public void VerifyV2_AddBaselinePerformanceTestRunResultsDirectory()
324324
public void VerifyV2_Verify_AddBaselineAndNonBaselinePerformanceTestRunResults()
325325
{
326326
// Arrange
327-
var resultJsonFilePath = EnsureFullPath("results2.Json");
328-
var baselineJsonFilePath = EnsureFullPath("baseline2.Json");
327+
var resultJsonFilePath = EnsureFullPath("results2.json");
328+
var baselineJsonFilePath = EnsureFullPath("baseline2.json");
329329
var args = new[]
330330
{
331331
"--format=Json",
@@ -381,5 +381,5 @@ private void AssertCorrectResultsJsonDirectoryPaths(string[] resultsJsonDirPaths
381381
}
382382
}
383383
}
384-
384+
385385
}

0 commit comments

Comments
 (0)