Skip to content

Commit fb07ce0

Browse files
author
Thomas Kiley
authored
Merge pull request #5248 from thk123/dont-run-failed-tests
Don't run failed unit tests by default
2 parents 1e6ce0b + 6176871 commit fb07ce0

File tree

14 files changed

+57
-18
lines changed

14 files changed

+57
-18
lines changed

‎.travis.yml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ jobs:
243243
- cmake -S . -Bbuild '-DCMAKE_BUILD_TYPE=Release' '-DCMAKE_CXX_COMPILER=/usr/bin/g++-7' '-DCMAKE_USE_CUDD=true' -DCMAKE_CXX_FLAGS="-DBDD_GUARDS" '-DWITH_MEMORY_ANALYZER=On'
244244
- git submodule update --init --recursive
245245
- cmake --build build -- -j4
246-
script: (cd build; ctest -V -L CORE -j2)
246+
script: (cd build; ctest -V -L CORE -j2; ctest -V -R unit-xfail -j2)
247247

248248
# cmake build using clang++-6
249249
- stage: Test different OS/CXX/Flags
@@ -280,7 +280,7 @@ jobs:
280280
- cmake -S . -Bbuild '-DCMAKE_BUILD_TYPE=Release' '-DCMAKE_CXX_COMPILER=/usr/bin/clang++-7' '-DCMAKE_CXX_FLAGS=-Qunused-arguments' '-DWITH_MEMORY_ANALYZER=On'
281281
- git submodule update --init --recursive
282282
- cmake --build build -- -j4
283-
script: (cd build; ctest -V -L CORE -j2)
283+
script: (cd build; ctest -V -L CORE -j2; ctest -V -R unit-xfail -j2)
284284

285285
# cmake build on OSX, using default clang
286286
- stage: Test different OS/CXX/Flags
@@ -300,7 +300,7 @@ jobs:
300300
- cmake -S . -Bbuild '-DCMAKE_BUILD_TYPE=Release' '-DCMAKE_OSX_ARCHITECTURES=x86_64'
301301
- git submodule update --init --recursive
302302
- cmake --build build -- -j4
303-
script: (cd build; ctest -V -L CORE -j2)
303+
script: (cd build; ctest -V -L CORE -j2; ctest -V -R unit-xfail -j2)
304304

305305

306306
# Run Coverity
@@ -355,9 +355,13 @@ script:
355355
- env PATH=$PATH:`pwd`/src/solvers UBSAN_OPTIONS=print_stacktrace=1 make -C regression/cbmc test-cprover-smt2
356356
- make -C unit "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2
357357
- make -C unit test
358+
- echo "Running expected failure tests"
359+
- make TAGS="[!shouldfail]" -C unit test
358360
- env UBSAN_OPTIONS=print_stacktrace=1 make -C jbmc/regression test-parallel "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2 JOBS=2
359361
- make -C jbmc/unit "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2
360362
- make -C jbmc/unit test
363+
- echo "Running expected failure tests"
364+
- make TAGS="[!shouldfail]" -C jbmc/unit test
361365

362366
before_cache:
363367
- ccache -s

‎buildspec-linux-clang.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,14 @@ phases:
2727
post_build:
2828
commands:
2929
- make -C unit test
30+
- echo "Running expected failure tests"
31+
- make TAGS="[!shouldfail]" -C unit test
3032
- make -C regression test CXX='ccache /usr/bin/clang++-8' CXX_FLAGS='-Qunused-arguments'
3133
- make -C regression/cbmc test-paths-lifo
3234
- env PATH=$PATH:`pwd`/src/solvers make -C regression/cbmc test-cprover-smt2
3335
- make -C jbmc/unit test
36+
- echo "Running expected failure tests"
37+
- make TAGS="[!shouldfail]" -C jbmc/unit test
3438
- make -C jbmc/regression test
3539
- echo Build completed on `date`
3640
cache:

‎buildspec-linux-cmake-gcc.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ phases:
2222
post_build:
2323
commands:
2424
- cd build; ctest -V -L CORE -j2
25+
- cd build; ctest -V -R unit-xfail -j2
2526
- echo Build completed on `date`
2627
cache:
2728
paths:

‎buildspec-windows.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ phases:
5757
$env:Path = "C:\tools\cygwin\bin;$env:Path"
5858
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make -C unit test BUILD_ENV=MSVC" '
5959
60+
- |
61+
$env:Path = "C:\tools\cygwin\bin;$env:Path"
62+
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make TAGS=[!shouldfail] -C unit test BUILD_ENV=MSVC" '
63+
6064
- |
6165
$env:Path = "C:\tools\cygwin\bin;$env:Path"
6266
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make -C jbmc/regression test BUILD_ENV=MSVC" '
@@ -65,6 +69,10 @@ phases:
6569
$env:Path = "C:\tools\cygwin\bin;$env:Path"
6670
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make -C jbmc/unit test BUILD_ENV=MSVC" '
6771
72+
- |
73+
$env:Path = "C:\tools\cygwin\bin;$env:Path"
74+
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make TAGS=[!shouldfail] -C jbmc/unit test BUILD_ENV=MSVC" '
75+
6876
cache:
6977
paths:
7078
- 'c:\clcache\**\*'

‎buildspec.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,14 @@ phases:
2626
post_build:
2727
commands:
2828
- make -C unit test
29+
- echo "Running expected failure tests"
30+
- make TAGS="[!shouldfail]" -C unit test
2931
- make -C regression test
3032
- make -C regression/cbmc test-paths-lifo
3133
- env PATH=$PATH:`pwd`/src/solvers make -C regression/cbmc test-cprover-smt2
3234
- make -C jbmc/unit test
35+
- echo "Running expected failure tests"
36+
- make TAGS="[!shouldfail]" -C jbmc/unit test
3337
- make -C jbmc/regression test
3438
- echo Build completed on `date`
3539
cache:

‎jbmc/unit/CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,11 @@ add_test(
3636
COMMAND $<TARGET_FILE:java-unit>
3737
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
3838
)
39+
40+
add_test(
41+
NAME java-unit-xfail
42+
COMMAND $<TARGET_FILE:java-unit> "[!shouldfail]"
43+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
44+
)
45+
3946
set_tests_properties(java-unit PROPERTIES LABELS "CORE;CBMC")

‎jbmc/unit/Makefile‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,9 +144,10 @@ all: $(CATCH_TEST)
144144
clean: java-testing-utils-clean
145145

146146
test: $(CATCH_TEST)
147-
if ! ./$(CATCH_TEST) -l | grep -q "^$(N_CATCH_TESTS) test cases" ; then \
148-
./$(CATCH_TEST) -l ; fi
149-
./$(CATCH_TEST)
147+
# Include hidden tests by specifying "*,[.]" for tests to count
148+
if ! ./$(CATCH_TEST) "*,[.]" -l | grep -q "^$(N_CATCH_TESTS) matching test cases" ; then \
149+
./$(CATCH_TEST) "*,[.]" -l ; fi
150+
./$(CATCH_TEST) ${TAGS}
150151

151152

152153
###############################################################################

‎jbmc/unit/java_bytecode/ci_lazy_methods/lazy_load_lambdas.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ Author: Diffblue Limited.
1212

1313
SCENARIO(
1414
"Lazy load lambda methods",
15-
"[core][java_bytecode][ci_lazy_methods][lambdas][!mayfail]")
15+
"[core][java_bytecode][ci_lazy_methods][lambdas]")
1616
{
1717
GIVEN("A class with some locally declared lambdas")
1818
{
@@ -192,7 +192,7 @@ SCENARIO(
192192

193193
SCENARIO(
194194
"Lazy load lambda methods in seperate class",
195-
"[core][java_bytecode][ci_lazy_methods][lambdas][!mayfail]")
195+
"[core][java_bytecode][ci_lazy_methods][lambdas]")
196196
{
197197
const symbol_tablet symbol_table = load_java_class_lazy(
198198
"ExternalLambdaAccessor",

‎jbmc/unit/java_bytecode/java_bytecode_convert_method/convert_invoke_dynamic.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ void validate_local_variable_lambda_assignment(
188188
SCENARIO(
189189
"Converting invokedynamic with a local lambda",
190190
"[core]"
191-
"[lambdas][java_bytecode][java_bytecode_convert_method][!mayfail]")
191+
"[lambdas][java_bytecode][java_bytecode_convert_method]" XFAIL)
192192
{
193193
// NOLINTNEXTLINE(whitespace/braces)
194194
run_test_with_compilers([](const std::string &compiler) {
@@ -380,7 +380,7 @@ void validate_member_variable_lambda_assignment(
380380
SCENARIO(
381381
"Converting invokedynamic with a member lambda",
382382
"[core]"
383-
"[lamdba][java_bytecode][java_bytecode_convert_method][!mayfail]")
383+
"[lamdba][java_bytecode][java_bytecode_convert_method]" XFAIL)
384384
{
385385
// NOLINTNEXTLINE(whitespace/braces)
386386
run_test_with_compilers([](const std::string &compiler) {
@@ -548,7 +548,7 @@ void validate_static_member_variable_lambda_assignment(
548548
SCENARIO(
549549
"Converting invokedynamic with a static member lambda",
550550
"[core]"
551-
"[lamdba][java_bytecode][java_bytecode_convert_method][!mayfail]")
551+
"[lamdba][java_bytecode][java_bytecode_convert_method]" XFAIL)
552552
{
553553
// NOLINTNEXTLINE(whitespace/braces)
554554
run_test_with_compilers([](const std::string &compiler) {

‎jbmc/unit/solvers/strings/string_constraint_instantiation/instantiate_not_contains.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,10 +172,9 @@ decision_proceduret::resultt check_sat(const exprt &expr, const namespacet &ns)
172172
return solver();
173173
}
174174

175-
// The [!mayfail] tag allows tests to fail while reporting the failure
176175
SCENARIO(
177176
"instantiate_not_contains",
178-
"[!mayfail][core][solvers][refinement][string_constraint_instantiation]")
177+
"[core][solvers][refinement][string_constraint_instantiation]" XFAIL)
179178
{
180179
// For printing expression
181180
register_language(new_java_bytecode_language);

0 commit comments

Comments
 (0)