From 430c76a33af4deff252c08b7a08abfda778a444b Mon Sep 17 00:00:00 2001 From: Olivier Bitter Date: Tue, 24 Mar 2026 19:13:58 +0100 Subject: [PATCH] feat(tests): add test-lib (tlib) feat(wrapperModules.direnv): init style(wrapperModues.direnv): apply nix fmt fix(wrapperModules.direnv): add maintainers feat: use key direnv: add checks refactor tests rework tests use a single drv for tests rework: use nix functions as assertions This way we have a way to join assertions using '&&' and can group assertions. If any assertion fails it will print the error message of the given assertin as well as the name of the test make all tests assertion-based rename functions add documentation add mise integration fix maintainers add env.CONFIG_DIRENV remove "key"-transform hack construcFiles sanitizes the key by default now add tlib cleanup pass wrapperModule to runTests (wip) add recursive test suites (wip) wrap in 'run' function fmt add indentBlock nice indents remove level param add busted syntax adapt runTest bckp cleanup fmt add debug option support passing str assertions rewrite direnv tests to set representation remove describe/it display test name in error log simplify render introduce errMsg bckp rework settings parsing rename testWrapper -> test getting there add debug flag use toSanitizedJSON use repeatStr from wlib use exit instead of return at top level handle multiline messages formatting document test-lib.test add example for wrapper tests formatting handle rendering multi-line conditions add docs improve writing Make comment nixdoc compatible generate nixdoc comments for the predefined assertions reformat documentation for 'test' Add contributing chapter to mdbook Apply suggestion from @BirdeeHub Co-authored-by: Birdee <85372418+BirdeeHub@users.noreply.github.com> Add enable option Co-authored-by: Birdee <85372418+BirdeeHub@users.noreply.github.com> fix enable option document enable option remove direnv module Will be added in a separate PR --- CONTRIBUTING.md | 140 ++++++++++++----- ci/docs/default.nix | 17 +++ ci/flake.nix | 3 +- ci/test-lib.nix | 328 ++++++++++++++++++++++++++++++++++++++++ maintainers/default.nix | 5 + 5 files changed, 456 insertions(+), 37 deletions(-) create mode 100644 ci/test-lib.nix diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da743c84..256cd120 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Adding Modules! +## Adding Modules! There are 2 kinds of modules in this repository. One kind which defines the `package` option, and one kind which does not. @@ -121,15 +121,15 @@ Example: } ``` -# Formatting +## Formatting `nix fmt` -# Tests +## Tests `nix flake check -Lv ./ci` -# Run Site Generator Locally +## Run Site Generator Locally `nix run ./ci` @@ -137,62 +137,130 @@ or `nix run ./ci#docs` -# Writing tests +To run the tests for an individual wrapper only, run + +`nix build ./ci#checks.{system}.wrapperModule-{name}` + +Example (neovim on `x86_64-linux`): + +`nix build ./ci#checks.x86_64-linux.wrapperModule-neovim` + +## Writing Tests You may also include a `check.nix` file in your module's directory. -It will be called via `pkgs.callPackage`, provided with the flake `self` value. -(i.e. `pkgs.callPackage your_check.nix { inherit self; }`) +It will be called via `pkgs.callPackage`, provided with the flake `self` value, as well as a test-library `tlib` value. +(i.e. `pkgs.callPackage your_check.nix { inherit self tlib; }`) -It should build a derivation which tests the wrapper derivation as best you can. -If a command fails, it fails the test. If it builds the derivation successfully, it passes the test. +We provide a testing library `tlib` that provides an easy-to-use interface to write tests. -If the program gives options for running the program to check the generated configuration is correct, you should do that. +### Writing Tests for Wrappers -Sometimes it is not easily possible to run the program within a derivation, in those cases, searching the wrapper derivation and other generated files and their contents is also acceptable. +If you are writing tests for a wrapper module, it is important to pass the name +of the wrapper to the first argument of the `test` function like in the example +below (marked at `(*)`). By doing this, we can grab the specified `wrapper.meta.platforms` config +of the wrapper (if any) and ensure that the tests are only run on the required platforms. -Example: ```nix { pkgs, - runCommand, self, + tlib, + ... }: + let - gitWrapped = self.wrappers.git.wrap { - inherit pkgs; - settings = { - user = { - name = "Test User"; - email = "test@example.com"; + inherit (tlib) + fileContains + isDirectory + isFile + notIsFile + areEqual + test + ; +in +test { wrapper = "direnv"; } { # <-- Specify the name of the wrapper here (*) + + "direnv wrapper should be created" = + let + wrapper = self.wrappers.direnv.wrap { + inherit pkgs; + nix-direnv.enable = true; }; - }; + in + [ + "[[ -d ${wrapper} ]]" # <-- a simple condition to be asserted + { + cond = "[[ -d ${wrapper} ]]"; + msg = "No directory found for wrapper."; # <-- you can also specify a custom error message + } + (isDirectory wrapper) # <-- or use pre-defined helpers + ]; + + "wrapper should output correct version" = + let + wrapper = self.wrappers.direnv.wrap { + inherit pkgs; + }; + in + '' # <-- no need to provide a list if there is only one assertion + "${wrapper}/bin/direnv" --version | + grep -q "${wrapper.version}" + ''; + + "math-tests" = { # <-- tests can be arbitrarily grouped + addition = [ + (areEqual 2 (1 + 1)) + (areEqual 7 (5 + 2)) + ]; + multiplication = [ + (areEqual 1 (1 * 1)) + (areEqual 10 (5 * 2)) + ]; }; -in -runCommand "git-test" { } '' - "${gitWrapped}/bin/git" config user.name | grep -q "Test User" - "${gitWrapped}/bin/git" config user.email | grep -q "test@example.com" - touch $out -'' +} ``` -If your module declares a list of valid platforms via its `meta.platforms` option, you should disable your test on the relevant platforms like so: + +Pre-defined assertions like `isDirectory` or `areEqual` are already available in tlib. +Feel free to contribute more if you find new ones that other maintainers might benefit from. + +### Writing Tests for Helper Modules or Library Functions + +The syntax is identical to [the example above](#writing-tests-for-wrappers), +except you don't provide a wrapper but a name: ```nix -if builtins.elem pkgs.stdenv.hostPlatform.system self.wrappers.waybar.meta.platforms then - pkgs.runCommand "waybar-test" { } '' - "${waybarWrapped}/bin/waybar" --version | grep -q "${waybarWrapped.version}" - touch $out - '' -else - null +{ + pkgs, + self, + tlib, + ... +}: + +let + inherit (tlib) + fileContains + isDirectory + isFile + notIsFile + areEqual + test + ; +in +test "my-test" { # <-- Specify an arbitrary name for your test +# test { name = "my-test" } { # <-- This is equivalent + + "my first test" = [ ... ]; # <-- nothing new here + "my second test" = [ ... ]; +} ``` If you are writing a helper module, or something very complex, you may wish to have multiple derivations. Simply return a set of them instead. -# Commit Messages +## Commit Messages Changes to wrapper modules should be titled `(wrapperModules.): some description`. For new additions, the description should be `init`, with any further explanation on subsequent lines @@ -212,6 +280,6 @@ For everything else, do the best you can to follow conventional commit message s Why specify this? I was having trouble figuring out what to title my commits. So now I know. -# Questions? +## Questions? The [github discussions board](https://github.com/BirdeeHub/nix-wrapper-modules/discussions) is open and a great place to find help! diff --git a/ci/docs/default.nix b/ci/docs/default.nix index e39ecc21..544655bd 100644 --- a/ci/docs/default.nix +++ b/ci/docs/default.nix @@ -208,6 +208,23 @@ in src = "${placeholder "generated"}/wrapper_docs/${n}.md"; }) (builtins.attrNames config.drv.wrapper_docs); } + { + name = "Contributing"; + data = "numbered"; + path = "md/CONTRIBUTING.md"; + src = ../../CONTRIBUTING.md; + subchapters = [ + { + name = "tlib"; + data = "numbered"; + path = "tlib.md"; + src = "${placeholder "out"}/wrappers-lib/tlib.md"; + build = '' + ${pkgs.nixdoc}/bin/nixdoc --category "" --description 'Testing library `tlib` documentation' --file '${../test-lib.nix}' --prefix "tlib" >> $out/wrappers-lib/tlib.md + ''; + } + ]; + } ]; }; } diff --git a/ci/flake.nix b/ci/flake.nix index d5740ac3..5d6e3965 100644 --- a/ci/flake.nix +++ b/ci/flake.nix @@ -20,6 +20,7 @@ inherit system; config.allowUnfree = true; }; + tlib = pkgs.callPackage ./test-lib.nix { inherit self; }; # Load checks from ci/checks/ directory coreAndCiChecks = lib.pipe ./checks [ @@ -43,7 +44,7 @@ name = "${prefix}-${name}"; inherit value; }; - result = pkgs.callPackage value { inherit self; }; + result = pkgs.callPackage value { inherit self tlib; }; in if result == null then [ ] diff --git a/ci/test-lib.nix b/ci/test-lib.nix new file mode 100644 index 00000000..297ea403 --- /dev/null +++ b/ci/test-lib.nix @@ -0,0 +1,328 @@ +{ + self, + lib, + runCommand, + writeShellScript, + stdenv, + pkgs, + ... +}: +let + wlib = self.lib; + + errMsg = msg: "(echo ${renderMsg msg} >&2 && return 1)"; + + indentBlock = + str: num: + let + idnt = " "; + lines = lib.splitString "\n" str; + indentLine = line: (wlib.repeatStr idnt num) + line; + in + lib.concatMapStringsSep "\n" indentLine lines; + + # Use [ANSI-C Quoting](https://www.gnu.org/software/bash/manual/html_node/ANSI_002dC-Quoting.html#ANSI_002dC-Quoting-1) + # to properly display multi-line messages without being indented if occurring in deeply nested tests. + renderMsg = + str: + let + lines = lib.splitString "\n" str; + result = "$'${lib.concatStringsSep "\\n" lines}'"; + in + if (lib.length lines) == 1 then str else result; + + toSanitizedJSON = + value: + if builtins.isAttrs value then + builtins.toJSON ( + lib.mapAttrsRecursive ( + path: v: + if builtins.isFunction v then + let + res = builtins.unsafeGetAttrPos (lib.last path) ( + lib.getAttrFromPath (lib.sublist 0 (builtins.length path - 1) path) value + ); + in + "" + else + v + ) value + ) + else + builtins.toJSON value; + + render = + node: + let + renderAssertion = + let + _renderAssertion = + { cond, msg }: + '' + ( + ${cond} + ) || ${errMsg msg}''; + in + assertion: + if lib.isString assertion then + _renderAssertion { + cond = assertion; + msg = "Failed assertion: ${assertion}"; + } + else + _renderAssertion assertion; + + renderAssertions = + assertions: lib.concatMapStringsSep " &&\n" (a: "(${renderAssertion a})") assertions; + + renderNode = + node: + let + block = + if !(lib.isAttrs node) then + renderAssertions (lib.toList node) + else + lib.concatMapAttrsStringSep " &&\n" (name: childNode: '' + (( + ${renderNode childNode} + ) || ${errMsg name})'') node; + in + indentBlock block 1; + in + renderNode node; + + _test = + { + name, + debug ? false, + }: + testSet: + let + testScript = '' + run () { + ${render testSet}; + } + run || (echo "Test '${name}' failed" >&2 && exit 1) + ''; + in + if debug then + writeShellScript name '' + ${testScript} + '' + else + runCommand name { passthru.test = testScript; } '' + ${testScript} + touch $out + ''; + +in +{ + /** + Takes tests provided as an attrs in `testSet`, renders it to a bash script, + and executes the rendered script using `pkgs.runCommand`. + + If the command runs successfully, the test set passes. Otherwise, a detailed + error message describing the failed assertion is shown in the logs. + + # Type + ``` + test :: (String | AttrSet) -> TestSet -> Derivation + ``` + + where: + + ``` + TestSet :: attrsOf (TestSet | Test) + + Test :: [ Assertion ] + + Assertion :: String | { cond :: String; msg :: String; } + ``` + + # Arguments + + settings + : Either a string or an attrs. + + If it is a string, it will be taken as the name of the derivation built + by `runCommand`. + + If it is a set, at least one of the keys `name` or `wrapper` need to be set + with a string value. + + - If you are defining tests for wrappers you should set `settings.wrapper` to the name + of the wrapper. This way, the `wrapper.meta.platforms` is considered during test execution, + and the test will only be run if the system it is running on is supported. + - If `settings.wrapper` is not set, the test will always be run. This makes sense if you + are testing core options or library functions. + - If only `settings.wrapper` is set, the name will be derived from this value by suffixing it with `-test`. + - If `settings.name` is set, it will be taken as the name of the derivation. + - If both are set, `settings.name` takes precedence. + + There is also an optional boolean option `settings.debug` (default = `false`). + If it is set to `true`, the generated bash script is built as an executable script that can be inspected + and run in `result` when running a specific test. + + The test can be disabled by setting `settings.enable = false`. + + testSet + : The set of tests to run. + */ + test = + settings: testSet: + let + name = + if lib.isString settings then + settings + else if settings ? name then + settings.name + else if settings ? wrapper then + "${settings.wrapper}-test" + else + throw '' + Invalid argument for `test`. + The first argument must be either a string (the test name) or an attrs + with at least one of the keys 'name' or 'wrapper', but got: + + ${lib.toSanitizedJSON settings} + ''; + wrapperName = settings.wrapper or null; + platforms = + if wrapperName == null then + null + else if (wrapperName != null) && (lib.hasAttr wrapperName self.wrappers) then + self.wrappers.${wrapperName}.meta.platforms + else + throw '' + Invalid argument for `test`. + The provided wrapper '${wrapperName}' was not found. + Available wrappers are: + + ${lib.toSanitizedJSON (lib.attrNames self.wrappers)} + ''; + enabled = + (settings.enable or true) + && (platforms == null || (platforms != null && builtins.elem stdenv.hostPlatform.system platforms)); + in + if enabled then + _test { + inherit name; + debug = settings.debug or false; + } testSet + else + null; + + /** + Returns an `Assertion` that checks whether `path` is an existing directory. + + # Type + ``` + isDirectory :: String -> Assertion + ``` + + # Arguments + path + : The filesystem path to check. + */ + isDirectory = path: { + cond = ''[[ -d "${path}" ]]''; + msg = "No such directory ${path}"; + }; + + /** + Returns an `Assertion` that checks whether `path` is an existing regular file. + + # Type + ``` + isFile :: String -> Assertion + ``` + + # Arguments + path + : The filesystem path to check. + */ + isFile = path: { + cond = ''[ -f "${path}" ]''; + msg = "No such file ${path}"; + }; + + /** + Returns an `Assertion` that checks whether `path` does **not** exist as a regular file. + + # Type + ``` + notIsFile :: String -> Assertion + ``` + + # Arguments + path + : The filesystem path that should be absent. + */ + notIsFile = path: { + cond = ''[ ! -f "${path}" ]''; + msg = "File ${path} should not exist"; + }; + + /** + Returns an `Assertion` that checks whether `file` contains a line matching `pattern`. + + The check is performed with `grep -q`, so `pattern` is treated as a basic regular expression. + + # Type + ``` + fileContains :: String -> String -> Assertion + ``` + + # Arguments + file + : Path to the file to search. + + pattern + : Basic regular expression to search for. + */ + fileContains = file: pattern: { + cond = ''grep -q '${pattern}' "${file}"''; + msg = "Pattern '${pattern}' not found in ${file}"; + }; + + /** + Returns an `Assertion` that checks whether `expected` and `actual` are equal. + + The comparison is performed in Nix at evaluation time. If the values differ, + the assertion message shows both values serialised as JSON. + + # Type + ``` + areEqual :: Any -> Any -> Assertion + ``` + + # Arguments + expected + : The expected value. + + actual + : The value to compare against `expected`. + */ + areEqual = + expected: actual: + let + equal = expected == actual; + in + { + cond = if equal then "true" else "false"; + msg = + if !equal then + '' + Expected: + ${toSanitizedJSON expected} + but got: + ${toSanitizedJSON actual}'' + else + "This should never happen."; + }; +} diff --git a/maintainers/default.nix b/maintainers/default.nix index b6bd9481..9d3b2773 100644 --- a/maintainers/default.nix +++ b/maintainers/default.nix @@ -61,6 +61,11 @@ githubId = 8916363; name = "Nikita Wootten"; }; + zenoli = { + name = "Zenoli"; + github = "zenoli"; + githubId = 8073528; + }; pengolord = { name = "pengo"; email = "pbalternates@gmail.com";