Skip to content

Commit 5e44cbf

Browse files
bpkrothbpkroth
authored
Improve test result checking (#427)
- Add checks on the result outputs to make sure that the length of time and number of queries expected to run were. - Make sure to exit non-zero when RuntimeExceptions are encountered. - Temporarily remove a faulty INSERT test from the templated benchmark. Split out from #283.
1 parent c0ef047 commit 5e44cbf

30 files changed

+270
-31
lines changed

‎.github/workflows/maven.yml‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -121,15 +121,17 @@ jobs:
121121
run: |
122122
# For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
123123
# In this case, we load the tpcc data.
124-
if [[ ${{matrix.benchmark}} == templated ]]; then
124+
if [[ ${{matrix.benchmark}} == templated ]] || [[ ${{matrix.benchmark}} == tpcc ]]; then
125125
# Disable synchronous mode for sqlite tpcc data loading to save some time.
126126
java -jar benchbase.jar -b tpcc -c config/sqlite/sample_tpcc_nosync_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
127127
# Run the templated benchmark.
128128
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
129129
else
130130
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/sqlite/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
131131
fi
132+
132133
# FIXME: Reduce the error rate so we don't need these overrides.
134+
results_benchmark=${{matrix.benchmark}}
133135
if [ ${{matrix.benchmark}} == auctionmark ]; then
134136
ERRORS_THRESHOLD=0.02
135137
elif [ ${{matrix.benchmark}} == resourcestresser ]; then
@@ -139,6 +141,7 @@ jobs:
139141
elif [ ${{matrix.benchmark}} == tatp ]; then
140142
ERRORS_THRESHOLD=0.05
141143
fi
144+
./scripts/check_latest_benchmark_results.sh $results_benchmark
142145
./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
143146
144147
## ----------------------------------------------------------------------------------
@@ -150,6 +153,7 @@ jobs:
150153
strategy:
151154
fail-fast: false
152155
matrix:
156+
# FIXME: Add tpch back in (#333).
153157
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
154158
services:
155159
mariadb: # https://hub.docker.com/_/mariadb
@@ -203,11 +207,13 @@ jobs:
203207
fi
204208
205209
# FIXME: Reduce the error rate so we don't need these overrides.
210+
results_benchmark=${{matrix.benchmark}}
206211
if [ ${{matrix.benchmark}} == auctionmark ]; then
207212
ERRORS_THRESHOLD=0.02
208213
elif [ ${{matrix.benchmark}} == tatp ]; then
209214
ERRORS_THRESHOLD=0.05
210215
fi
216+
./scripts/check_latest_benchmark_results.sh $results_benchmark
211217
./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
212218
213219
## ----------------------------------------------------------------------------------
@@ -219,7 +225,7 @@ jobs:
219225
strategy:
220226
fail-fast: false
221227
matrix:
222-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
228+
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
223229
services:
224230
mysql: # https://hub.docker.com/_/mysql
225231
image: mysql:latest
@@ -271,11 +277,13 @@ jobs:
271277
fi
272278
273279
# FIXME: Reduce the error rate so we don't need these overrides.
280+
results_benchmark=${{matrix.benchmark}}
274281
if [ ${{matrix.benchmark}} == auctionmark ]; then
275282
ERRORS_THRESHOLD=0.02
276283
elif [ ${{matrix.benchmark}} == tatp ]; then
277284
ERRORS_THRESHOLD=0.05
278285
fi
286+
./scripts/check_latest_benchmark_results.sh $results_benchmark
279287
./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
280288
281289
## ----------------------------------------------------------------------------------
@@ -287,7 +295,7 @@ jobs:
287295
strategy:
288296
fail-fast: false
289297
matrix:
290-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'twitter', 'voter', 'wikipedia', 'ycsb', 'templated' ]
298+
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb', 'templated' ]
291299
services:
292300
oracle:
293301
image: gvenzl/oracle-xe:21.3.0-slim-faststart
@@ -341,7 +349,9 @@ jobs:
341349
else
342350
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/oracle/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
343351
fi
352+
344353
# FIXME: Reduce the error rate so we don't need these overrides.
354+
results_benchmark=${{matrix.benchmark}}
345355
if [ ${{matrix.benchmark}} == auctionmark ]; then
346356
ERRORS_THRESHOLD=0.04
347357
elif [ ${{matrix.benchmark}} == tatp ]; then
@@ -353,6 +363,7 @@ jobs:
353363
elif [ ${{matrix.benchmark}} == wikipedia ]; then
354364
ERRORS_THRESHOLD=0.02
355365
fi
366+
./scripts/check_latest_benchmark_results.sh $results_benchmark
356367
./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
357368
358369
## ----------------------------------------------------------------------------------
@@ -414,11 +425,13 @@ jobs:
414425
fi
415426
416427
# FIXME: Reduce the error rate so we don't need these overrides.
428+
results_benchmark=${{matrix.benchmark}}
417429
if [ ${{matrix.benchmark}} == auctionmark ]; then
418430
ERRORS_THRESHOLD=0.02
419431
elif [ ${{matrix.benchmark}} == tatp ]; then
420432
ERRORS_THRESHOLD=0.05
421433
fi
434+
./scripts/check_latest_benchmark_results.sh $results_benchmark
422435
./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
423436
424437
## ----------------------------------------------------------------------------------
@@ -430,7 +443,7 @@ jobs:
430443
strategy:
431444
fail-fast: false
432445
matrix:
433-
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
446+
benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'templated', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
434447
services:
435448
cockroach: # https://hub.docker.com/repository/docker/timveil/cockroachdb-single-node
436449
image: timveil/cockroachdb-single-node:latest
@@ -461,14 +474,23 @@ jobs:
461474

462475
- name: Run benchmark
463476
run: |
464-
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
477+
# For templated benchmarks, we need to preload some data for the test since by design, templated benchmarks do not support the 'load' operation
478+
# In this case, we load the tpcc data.
479+
if [[ ${{matrix.benchmark}} == templated ]]; then
480+
java -jar benchbase.jar -b tpcc -c config/cockroachdb/sample_tpcc_config.xml --create=true --load=true --execute=false --json-histograms results/histograms.json
481+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=false --load=false --execute=true --json-histograms results/histograms.json
482+
else
483+
java -jar benchbase.jar -b ${{matrix.benchmark}} -c config/cockroachdb/sample_${{matrix.benchmark}}_config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
484+
fi
465485
# FIXME: Reduce the error rate so we don't need these overrides.
466486
# FIXME: Reduce the error rate so we don't need these overrides.
487+
results_benchmark=${{matrix.benchmark}}
467488
if [ ${{matrix.benchmark}} == auctionmark ]; then
468489
ERRORS_THRESHOLD=0.02
469490
elif [ ${{matrix.benchmark}} == tatp ]; then
470491
ERRORS_THRESHOLD=0.05
471492
fi
493+
./scripts/check_latest_benchmark_results.sh $results_benchmark
472494
./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
473495
474496
## ----------------------------------------------------------------------------------
@@ -480,8 +502,7 @@ jobs:
480502
strategy:
481503
fail-fast: false
482504
matrix:
483-
# TODO: add more benchmarks
484-
#benchmark: [ 'auctionmark', 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'seats', 'sibench', 'smallbank', 'tatp', 'tpcc', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
505+
# TODO: add auctionmark and seats benchmark
485506
benchmark: [ 'epinions', 'hyadapt', 'noop', 'otmetrics', 'resourcestresser', 'sibench', 'smallbank', 'tatp', 'tpcc', 'templated', 'tpch', 'twitter', 'voter', 'wikipedia', 'ycsb' ]
486507
services:
487508
sqlserver:
@@ -553,9 +574,11 @@ jobs:
553574
fi
554575
555576
# FIXME: Reduce the error rate so we don't need these overrides.
577+
results_benchmark=${{matrix.benchmark}}
556578
if [ ${{matrix.benchmark}} == tatp ]; then
557579
ERRORS_THRESHOLD=0.05
558580
fi
581+
./scripts/check_latest_benchmark_results.sh $results_benchmark
559582
./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
560583
561584
## ----------------------------------------------------------------------------------
@@ -632,6 +655,7 @@ jobs:
632655
"$image" -b "$benchmark" -c /tmp/config.xml --create=true --load=true --execute=true --json-histograms results/histograms.json
633656
# Test that the results files were produced.
634657
ls results/${benchmark}_*.csv
658+
./scripts/check_latest_benchmark_results.sh ${benchmark}
635659
./scripts/check_histogram_results.sh results/histograms.json $ERRORS_THRESHOLD
636660
done
637661
# Publish the docker image if the build/test was successful.

‎config/cockroachdb/sample_templated_config.xml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<?xml version="1.0"?>
2+
<parameters>
3+
4+
<!-- Connection details -->
5+
<type>COCKROACHDB</type>
6+
<driver>org.postgresql.Driver</driver>
7+
<url>jdbc:postgresql://localhost:26257/benchbase?sslmode=disable&amp;ApplicationName=templated&amp;reWriteBatchedInserts=true</url>
8+
<username>root</username>
9+
<password></password>
10+
<isolation>TRANSACTION_SERIALIZABLE</isolation>
11+
<batchsize>128</batchsize>
12+
13+
<!-- Note: these example template reuses TPC-C queries and data as a demonstration, but others are possible.
14+
To test this sample as is, you must first run benchbase with parameters `load=true` and
15+
`config=sample_tpcc_config.xml` to load the TPC-C data to the target database.
16+
Other templated benchmarks need to ensure that the data is loaded manually prior to running the benchmark.
17+
-->
18+
19+
<query_templates_file>data/templated/example.xml</query_templates_file>
20+
21+
<!-- The workload -->
22+
<terminals>1</terminals>
23+
<works>
24+
<work>
25+
<time>10</time>
26+
<rate>100</rate>
27+
<weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
28+
</work>
29+
</works>
30+
31+
<!-- Select all templates that are executed. -->
32+
<transactiontypes>
33+
<transactiontype>
34+
<name>GetOrder</name>
35+
</transactiontype>
36+
<transactiontype>
37+
<name>GetCust</name>
38+
</transactiontype>
39+
<transactiontype>
40+
<name>GetCustNull</name>
41+
</transactiontype>
42+
<transactiontype>
43+
<name>GetWarehouse</name>
44+
</transactiontype>
45+
<transactiontype>
46+
<name>GetItemByPrice</name>
47+
</transactiontype>
48+
<transactiontype>
49+
<name>UpdateItemPrice</name>
50+
</transactiontype>
51+
<transactiontype>
52+
<name>DeleteItem</name>
53+
</transactiontype>
54+
<!-- FIXME:
55+
<transactiontype>
56+
<name>InsertItem</name>
57+
</transactiontype>
58+
-->
59+
</transactiontypes>
60+
</parameters>

‎config/mariadb/sample_templated_config.xml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<work>
2525
<time>10</time>
2626
<rate>100</rate>
27-
<weights>20,20,10,10,10,10,10,10</weights>
27+
<weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
2828
</work>
2929
</works>
3030

@@ -51,8 +51,10 @@
5151
<transactiontype>
5252
<name>DeleteItem</name>
5353
</transactiontype>
54+
<!-- FIXME:
5455
<transactiontype>
5556
<name>InsertItem</name>
5657
</transactiontype>
58+
-->
5759
</transactiontypes>
5860
</parameters>

‎config/mysql/sample_templated_config.xml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<work>
2525
<time>10</time>
2626
<rate>100</rate>
27-
<weights>20,20,10,10,10,10,10,10</weights>
27+
<weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
2828
</work>
2929
</works>
3030

@@ -51,8 +51,10 @@
5151
<transactiontype>
5252
<name>DeleteItem</name>
5353
</transactiontype>
54+
<!-- FIXME:
5455
<transactiontype>
5556
<name>InsertItem</name>
5657
</transactiontype>
58+
-->
5759
</transactiontypes>
5860
</parameters>

‎config/oracle/sample_templated_config.xml‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<work>
2525
<time>10</time>
2626
<rate>100</rate>
27-
<weights>30,20,10,30,10</weights>
27+
<weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
2828
</work>
2929
</works>
3030

@@ -45,5 +45,16 @@
4545
<transactiontype>
4646
<name>GetItemByPrice</name>
4747
</transactiontype>
48+
<transactiontype>
49+
<name>UpdateItemPrice</name>
50+
</transactiontype>
51+
<transactiontype>
52+
<name>DeleteItem</name>
53+
</transactiontype>
54+
<!-- FIXME:
55+
<transactiontype>
56+
<name>InsertItem</name>
57+
</transactiontype>
58+
-->
4859
</transactiontypes>
4960
</parameters>

‎config/postgres/sample_templated_config.xml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<work>
2525
<time>10</time>
2626
<rate>100</rate>
27-
<weights>20,20,10,10,10,10,10,10</weights>
27+
<weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
2828
</work>
2929
</works>
3030

@@ -51,8 +51,10 @@
5151
<transactiontype>
5252
<name>DeleteItem</name>
5353
</transactiontype>
54+
<!-- FIXME:
5455
<transactiontype>
5556
<name>InsertItem</name>
5657
</transactiontype>
58+
-->
5759
</transactiontypes>
5860
</parameters>

‎config/sqlite/sample_templated_config.xml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
<work>
2323
<time>10</time>
2424
<rate>100</rate>
25-
<weights>20,20,10,10,10,10,10,10</weights>
25+
<weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
2626
</work>
2727
</works>
2828

@@ -49,8 +49,10 @@
4949
<transactiontype>
5050
<name>DeleteItem</name>
5151
</transactiontype>
52+
<!-- FIXME:
5253
<transactiontype>
5354
<name>InsertItem</name>
5455
</transactiontype>
56+
-->
5557
</transactiontypes>
5658
</parameters>

‎config/sqlserver/sample_hyadapt_config.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,6 @@
114114
<transactiontype>
115115
<name>SumRecord10</name>
116116
</transactiontype>
117-
117+
118118
</transactiontypes>
119119
</parameters>

‎config/sqlserver/sample_templated_config.xml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<work>
2525
<time>10</time>
2626
<rate>100</rate>
27-
<weights>20,20,10,10,10,10,10,10</weights>
27+
<weights>20,20,10,10,10,10,10<!-- FIXME: ,10 --></weights>
2828
</work>
2929
</works>
3030

@@ -51,8 +51,10 @@
5151
<transactiontype>
5252
<name>DeleteItem</name>
5353
</transactiontype>
54+
<!-- FIXME:
5455
<transactiontype>
5556
<name>InsertItem</name>
5657
</transactiontype>
58+
-->
5759
</transactiontypes>
5860
</parameters>

‎config/sqlserver/sample_ycsb_config.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
<!-- Scalefactor in YCSB is *1000 the number of rows in the USERTABLE-->
1414
<scalefactor>1</scalefactor>
15-
15+
1616
<!-- Optional: Override the field size for each column in USERTABLE -->
1717
<!-- <fieldSize>8</fieldSize> -->
1818

0 commit comments

Comments
 (0)