Skip to content

Commit 9667400

Browse files
Terracom12Terracom12
committed
Add a filtering option for test cases (#61)
* Add --filter cli arg * Make use of filter in test runners for both prof and student * add using namespace std::literals directive to main header * chore: set number of jobs for builds/tests to nproc/2 * chore: set jobs to max for ci * chore: colorize makefile list-ops action * fix: remove invalid -j flag to cmake configure * chore: try ci with reduced job count, see if tests pass
1 parent 7436700 commit 9667400

11 files changed

Lines changed: 61 additions & 20 deletions

File tree

‎.github/workflows/docs.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222

2323
- name: Build Docs
2424
run: |
25-
make build-docs
25+
make build-docs NUM_JOBS=$(nproc)
2626
2727
- name: Setup GitHub Pages
2828
uses: actions/configure-pages@v5

‎Makefile‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ CMAKE_CONFIGURE_EXTRA_ARGS ?=
1616
CMAKE_BUILD_EXTRA_ARGS ?=
1717
# gcc or clang
1818
COMPILER ?= gcc
19+
NUM_JOBS ?= $(shell echo $$(( $$(nproc) / 2 )))
1920

2021
ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
2122

@@ -28,6 +29,7 @@ DOCS_PRESET := unixlike-docs-only
2829

2930
SRC_ENV := if [ -f "$(ROOT_DIR)/.env" ]; then export $$(cat "$(ROOT_DIR)/.env" | xargs); echo "Set enviornment variables:"; sed -E 's/=.*//' "$(ROOT_DIR)/.env"; echo; fi
3031

32+
3133
default: help
3234

3335
##### --- snipped from gh:compiler-explorer/compiler-explorer/blob/main/Makefile
@@ -64,34 +66,34 @@ build: build-debug ## alias for build-debug
6466

6567
.PHONY: build-debug
6668
build-debug: configure-debug ## build in debug mode
67-
@$(SRC_ENV) && cmake --build --preset $(DEBUG_PRESET) $(CMAKE_BUILD_EXTRA_ARGS)
69+
@$(SRC_ENV) && cmake --build --preset $(DEBUG_PRESET) $(CMAKE_BUILD_EXTRA_ARGS) -j $(NUM_JOBS)
6870

6971
.PHONY: build-release
7072
build-release: configure-release ## build in release mode (with debug info)
71-
@$(SRC_ENV) && cmake --build --preset $(RELEASE_PRESET) $(CMAKE_BUILD_EXTRA_ARGS)
73+
@$(SRC_ENV) && cmake --build --preset $(RELEASE_PRESET) $(CMAKE_BUILD_EXTRA_ARGS) -j $(NUM_JOBS)
7274

7375
.PHONY: build-docs
7476
build-docs: configure-docs ## build in release mode (with debug info)
75-
@$(SRC_ENV) && cmake --build --preset $(DOCS_PRESET) $(CMAKE_BUILD_EXTRA_ARGS)
77+
@$(SRC_ENV) && cmake --build --preset $(DOCS_PRESET) $(CMAKE_BUILD_EXTRA_ARGS) -j $(NUM_JOBS)
7678

7779
.PHONY: build-tests-debug
7880
build-tests-debug: configure-debug
79-
@$(SRC_ENV) && cmake --build --preset $(DEBUG_PRESET) --target asmgrader_tests $(CMAKE_BUILD_EXTRA_ARGS)
81+
@$(SRC_ENV) && cmake --build --preset $(DEBUG_PRESET) --target asmgrader_tests $(CMAKE_BUILD_EXTRA_ARGS) -j $(NUM_JOBS)
8082

8183
.PHONY: build-tests-release
8284
build-tests-release: configure-release
83-
@$(SRC_ENV) && cmake --build --preset $(RELEASE_PRESET) --target asmgrader_tests $(CMAKE_BUILD_EXTRA_ARGS)
85+
@$(SRC_ENV) && cmake --build --preset $(RELEASE_PRESET) --target asmgrader_tests $(CMAKE_BUILD_EXTRA_ARGS) -j $(NUM_JOBS)
8486

8587
.PHONY: test
8688
test: test-debug ## alias for test-debug
8789

8890
.PHONY: test-debug
8991
test-debug: build-tests-debug ## run tests in debug mode
90-
@$(SRC_ENV) && ctest --preset $(DEBUG_PRESET) --progress --output-on-failure --no-tests=error $(CTEST_EXTRA_ARGS)
92+
@$(SRC_ENV) && ctest --preset $(DEBUG_PRESET) --progress --output-on-failure --no-tests=error $(CTEST_EXTRA_ARGS) -j $(NUM_JOBS)
9193

9294
.PHONY: test-release
9395
test-release: build-tests-release ## run tests in release mode
94-
@$(SRC_ENV) && ctest --preset $(RELEASE_PRESET) --progress --output-on-failure --no-tests=error $(CTEST_EXTRA_ARGS)
96+
@$(SRC_ENV) && ctest --preset $(RELEASE_PRESET) --progress --output-on-failure --no-tests=error $(CTEST_EXTRA_ARGS) -j $(NUM_JOBS)
9597

9698

9799
.PHONY: clean
@@ -107,8 +109,10 @@ deep-clean: ## remove all build files and configuration
107109

108110
.PHONY: list-opts
109111
list-opts: ## list available CMake options
110-
@if ! cmake -S . -B build/ -LAH -N 2>/dev/null \
111-
| grep ASMGRADER_ -B 1 \
112+
@if ! cmake -S . -B $(shell find build/ -maxdepth 1 -mindepth 1 | head -n1) -LAH -N 2>/dev/null \
113+
| grep ASMGRADER_ --before-context 1 --group-separator='' \
114+
| sed '/\/\// { s/^//; s/$$//; }' \
115+
| sed '/^\w/ { s/^//; s/:/:/; }' \
112116
; then \
113117
printf '\033[31mError: CMake command failed. Is the project configured?\033[0m\n'; \
114118
exit 1; \

‎include/asmgrader/asmgrader.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,5 +42,7 @@
4242
#include <variant> // IWYU pragma: export
4343
#include <vector> // IWYU pragma: export
4444

45+
using namespace std::literals;
46+
4547
// should always include last if possible, as the short macro names may conflict with other libraries.
4648
#include <asmgrader/api/test_macros.hpp> // IWYU pragma: export

‎src/app/professor_app.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ int ProfessorApp::run_impl() {
7171
std::shared_ptr output_serializer =
7272
std::make_shared<PlainTextSerializer>(output_sink, OPTS.colorize_option, OPTS.verbosity);
7373

74-
MultiStudentRunner runner{*assignment, output_serializer};
74+
MultiStudentRunner runner{*assignment, output_serializer, OPTS.tests_filter};
7575

7676
output_serializer->on_run_metadata(RunMetadata{});
7777

‎src/app/student_app.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ int StudentApp::run_impl() {
3131
StdoutSink output_sink;
3232
std::shared_ptr output_serializer =
3333
std::make_shared<PlainTextSerializer>(output_sink, OPTS.colorize_option, OPTS.verbosity);
34-
AssignmentTestRunner runner{*assignment, output_serializer};
34+
AssignmentTestRunner runner{*assignment, output_serializer, OPTS.tests_filter};
3535

3636
output_serializer->on_run_metadata(RunMetadata{});
3737
AssignmentResult res = runner.run_all(OPTS.file_name);

‎src/multi_student_runner.cpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,23 @@
88
#include <fmt/compile.h>
99

1010
#include <memory>
11+
#include <optional>
12+
#include <string>
1113
#include <utility>
1214
#include <vector>
1315

1416
namespace asmgrader {
1517

16-
MultiStudentRunner::MultiStudentRunner(Assignment& assignment, const std::shared_ptr<Serializer>& serializer)
18+
MultiStudentRunner::MultiStudentRunner(Assignment& assignment, const std::shared_ptr<Serializer>& serializer,
19+
const std::optional<std::string>& tests_filter)
1720
: assignment_{&assignment}
18-
, serializer_{serializer} {}
21+
, serializer_{serializer}
22+
, filter_{tests_filter} {}
1923

2024
MultiStudentResult MultiStudentRunner::run_all_students(const std::vector<StudentInfo>& students) const {
2125
MultiStudentResult result;
2226

23-
AssignmentTestRunner assignment_runner{*assignment_, serializer_};
27+
AssignmentTestRunner assignment_runner{*assignment_, serializer_, filter_};
2428

2529
for (const StudentInfo& info : students) {
2630
serializer_->on_student_begin(info);

‎src/multi_student_runner.hpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,24 @@
55
#include "output/serializer.hpp"
66

77
#include <memory>
8+
#include <optional>
9+
#include <string>
810
#include <vector>
911

1012
namespace asmgrader {
1113

1214
class MultiStudentRunner
1315
{
1416
public:
15-
MultiStudentRunner(Assignment& assignment, const std::shared_ptr<Serializer>& serializer);
17+
MultiStudentRunner(Assignment& assignment, const std::shared_ptr<Serializer>& serializer,
18+
const std::optional<std::string>& tests_filter);
1619

1720
MultiStudentResult run_all_students(const std::vector<StudentInfo>& students) const;
1821

1922
private:
2023
Assignment* assignment_;
2124
std::shared_ptr<Serializer> serializer_;
25+
std::optional<std::string> filter_;
2226
};
2327

2428
} // namespace asmgrader

‎src/test_runner.cpp‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "program/program.hpp"
1111
#include "version.hpp"
1212

13+
#include <range/v3/view/filter.hpp>
1314
#include <range/v3/view/map.hpp>
1415

1516
#include <filesystem>
@@ -23,9 +24,11 @@
2324

2425
namespace asmgrader {
2526

26-
AssignmentTestRunner::AssignmentTestRunner(Assignment& assignment, const std::shared_ptr<Serializer>& serializer)
27+
AssignmentTestRunner::AssignmentTestRunner(Assignment& assignment, const std::shared_ptr<Serializer>& serializer,
28+
const std::optional<std::string>& tests_filter)
2729
: assignment_{&assignment}
28-
, serializer_{serializer} {}
30+
, serializer_{serializer}
31+
, filter_{tests_filter} {}
2932

3033
AssignmentResult AssignmentTestRunner::run_all(std::optional<std::filesystem::path> alternative_path) const {
3134
// Assignment name -> TestResults
@@ -36,7 +39,15 @@ AssignmentResult AssignmentTestRunner::run_all(std::optional<std::filesystem::pa
3639
assignment_->set_exec_path(std::move(*alternative_path));
3740
}
3841

39-
for (TestBase& test : assignment_->get_tests()) {
42+
auto maybe_tests_filter = ranges::views::filter([this](const TestBase& test) -> bool {
43+
if (!filter_.has_value()) {
44+
return true;
45+
}
46+
47+
return test.get_name().find(*filter_) != std::string::npos;
48+
});
49+
50+
for (TestBase& test : assignment_->get_tests() | maybe_tests_filter) {
4051
// Skip tests that are marked as professor-only if we're not in professor mode
4152
if (test.get_is_prof_only() && APP_MODE != AppMode::Professor) {
4253
continue;

‎src/test_runner.hpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,16 @@
77
#include <filesystem>
88
#include <memory>
99
#include <optional>
10+
#include <string>
1011

1112
namespace asmgrader {
1213

1314
/// Manages test execution and result aggregation for a specific assignment
1415
class AssignmentTestRunner
1516
{
1617
public:
17-
AssignmentTestRunner(Assignment& assignment, const std::shared_ptr<Serializer>& serializer);
18+
AssignmentTestRunner(Assignment& assignment, const std::shared_ptr<Serializer>& serializer,
19+
const std::optional<std::string>& tests_filter);
1820

1921
AssignmentResult run_all(std::optional<std::filesystem::path> alternative_path) const;
2022

@@ -23,6 +25,7 @@ class AssignmentTestRunner
2325

2426
Assignment* assignment_;
2527
std::shared_ptr<Serializer> serializer_;
28+
std::optional<std::string> filter_;
2629
};
2730

2831
} // namespace asmgrader

‎src/user/cl_args.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,14 @@ void CommandLineArgs::setup_parser() {
141141
.help("Whether/when to stop early, to not flood the console with failing test messages.");
142142

143143

144+
arg_parser_.add_argument("--filter")
145+
.metavar("STR")
146+
.nargs(1)
147+
.action([&] (const std::string& opt) {
148+
opts_buffer_.tests_filter = opt;
149+
})
150+
.help("Filter for test cases to be run. Matching occurs if STR occurs anywhere within the test case name.");
151+
144152
arg_parser_.add_argument("-c", "--color")
145153
.choices("never", "auto", "always")
146154
.default_value(std::string{"auto"})

0 commit comments

Comments
 (0)