Support for running tests with coverage

Author: makekryl

CMakeLists.txt

@@ -12,6 +12,7 @@ include(python)
 include(externals)
 include(diagnostics)
 include(testing)
+include(coverage)
 
 enable_testing()
 
@@ -21,3 +22,4 @@ add_subdirectory(bin)
 add_subdirectory(test)
 
 add_tests_build_target()
+enable_coverage_if_used()

cmake/coverage.cmake

@@ -0,0 +1,30 @@
+function(enable_coverage_if_used)
+  if(NOT VANADIUM_USE_COVERAGE)
+    return()
+  endif()
+
+  # TODO: extract to a separate aux .cmake file if one will be needed
+  function(collect_targets dir output)
+    get_property(local_targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS)
+    get_property(subdirs DIRECTORY ${dir} PROPERTY SUBDIRECTORIES)
+    foreach(subdir ${subdirs})
+      collect_targets(${subdir} sub_targets)
+      list(APPEND local_targets ${sub_targets})
+    endforeach()
+    set(${output} ${local_targets} PARENT_SCOPE)
+  endfunction()
+  collect_targets(${CMAKE_SOURCE_DIR} all_targets)
+
+  foreach(t ${all_targets})
+    get_target_property(type ${t} TYPE)
+    if(NOT type MATCHES "EXECUTABLE|STATIC_LIBRARY|SHARED_LIBRARY|MODULE_LIBRARY|OBJECT_LIBRARY")
+      continue()
+    endif()
+
+    get_target_property(dir ${t} SOURCE_DIR)
+    if(dir MATCHES "^${PROJECT_SOURCE_DIR}")
+      target_compile_options(${t} PRIVATE --coverage)
+      target_link_options(${t} PRIVATE --coverage)
+    endif()
+  endforeach()
+endfunction()

inv/params/build.py

@@ -10,7 +10,10 @@
 @dataclass(slots=True)
 class BuildOptions:
   toolchain: str = DEFAULT_TOOLCHAIN
+
   sanitizers: bool = False
+  coverage: bool = False
+
   release: bool = False
   static: bool = False
 

inv/params/test.py

@@ -9,10 +9,15 @@
 @dataclass(slots=True)
 class TestOptions:
   jobs: int | None = None
+
   filter: str | None = None
   exclude: str | None = None
+
   skip_build: bool = False
   no_run: bool = False
+
+  report_coverage: bool = False
+
   ctest_args: str | None = None
 
 

inv/tasks/build.py

@@ -50,8 +50,20 @@ def configure(
 
   use_compile_commands = True
 
+  # convenience thing: interesting flags that impact build are to be put in defs, flags that does not to be put in env
+  defs = {
+    "CMAKE_GENERATOR": "Ninja",
+  }
+  if build_opts(c).coverage:
+    # way too specific to add a separate preset for it
+    defs["VANADIUM_USE_COVERAGE"] = _cmake_bool(True)
+
   c.run(
-    f"cmake -DCMAKE_GENERATOR=Ninja --preset '{preset}' -B '{build_dir}'",
+    f"cmake"
+    f" {' '.join(f'-D{k}={v}' for k, v in defs.items())}"
+    f" --preset '{preset}'"
+    f" -B '{build_dir}'",
+    #
     env={
       "CMAKE_EXPORT_COMPILE_COMMANDS": _cmake_bool(use_compile_commands),
       "CMAKE_COLOR_DIAGNOSTICS": _cmake_bool(sys.stdout.isatty()),

inv/tasks/test.py

@@ -1,11 +1,17 @@
+import os
+import shutil
+
 from invoke import Context, task
 
+from inv.config import OUTPUT_DIR
 from inv.params import override_params_defaults
-from inv.params.build import with_build_params
+from inv.params.build import build_opts, with_build_params
 from inv.params.test import test_opts, with_test_params
 
 from . import build
 
+COVERAGE_DIR = OUTPUT_DIR / "coverage"
+
 
 @task(default=True)
 @override_params_defaults(sanitizers=True)
@@ -31,9 +37,39 @@ def test(c: Context, label: str):
     args.append(test_opts(c).ctest_args)
 
   c.run(
-    f"ctest -L '{label}' --output-on-failure --test-dir '{str(build_dir)}' {' '.join(args)}"
+    f"ctest"
+    f" -L '{label}'"
+    f" --output-on-failure"
+    f" --test-dir '{str(build_dir)}'"
+    f" {' '.join(args)}"
   )
 
+  if test_opts(c).report_coverage:
+    if COVERAGE_DIR.exists():
+      old_coverage_dir = COVERAGE_DIR.with_suffix(".old")
+      if old_coverage_dir.exists():
+        shutil.rmtree(old_coverage_dir)
+      os.rename(COVERAGE_DIR, old_coverage_dir)
+    COVERAGE_DIR.mkdir()
+
+    excluded_src_dirs = {
+      build_dir / "_deps",
+    }
+    excluded_src_dirs_args = " ".join(f"--exclude '{dir}'" for dir in excluded_src_dirs)
+
+    gcov_exec = "llvm-cov gcov" if build_opts(c).toolchain == "clang" else "gcov"
+    c.run(
+      f"gcovr"
+      f" --root .."
+      f" --object-directory '{build_dir}'"
+      f" --gcov-executable '{gcov_exec}'"
+      f" {excluded_src_dirs_args}"
+      f" --print-summary"
+      f" --html --html-details -o '{COVERAGE_DIR}/index.html'"
+      f" -j {test_opts(c).jobs}"
+    )
+    print(f"Coverage report is available at '{COVERAGE_DIR}'")
+
 
 @task
 @override_params_defaults(sanitizers=True)
@@ -52,3 +88,13 @@ def e2e(
   overwrite_snapshots: bool = False,
 ):
   test(c, label="e2e")
+
+
+@task
+def serve_coverage(c: Context):
+  if not COVERAGE_DIR.exists():
+    print(f"Coverage report not found at '{COVERAGE_DIR}'")
+    exit(1)  # TODO(inv): add something like failure(err_msg) that exits
+    return
+
+  c.run(f"python3 -m http.server -d '{COVERAGE_DIR}'")