diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da743c84..256cd120 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Adding Modules! +## Adding Modules! There are 2 kinds of modules in this repository. One kind which defines the `package` option, and one kind which does not. @@ -121,15 +121,15 @@ Example: } ``` -# Formatting +## Formatting `nix fmt` -# Tests +## Tests `nix flake check -Lv ./ci` -# Run Site Generator Locally +## Run Site Generator Locally `nix run ./ci` @@ -137,62 +137,130 @@ or `nix run ./ci#docs` -# Writing tests +To run the tests for an individual wrapper only, run + +`nix build ./ci#checks.{system}.wrapperModule-{name}` + +Example (neovim on `x86_64-linux`): + +`nix build ./ci#checks.x86_64-linux.wrapperModule-neovim` + +## Writing Tests You may also include a `check.nix` file in your module's directory. -It will be called via `pkgs.callPackage`, provided with the flake `self` value. -(i.e. `pkgs.callPackage your_check.nix { inherit self; }`) +It will be called via `pkgs.callPackage`, provided with the flake `self` value, as well as a test-library `tlib` value. +(i.e. `pkgs.callPackage your_check.nix { inherit self tlib; }`) -It should build a derivation which tests the wrapper derivation as best you can. -If a command fails, it fails the test. If it builds the derivation successfully, it passes the test. +We provide a testing library `tlib` that provides an easy-to-use interface to write tests. -If the program gives options for running the program to check the generated configuration is correct, you should do that. +### Writing Tests for Wrappers -Sometimes it is not easily possible to run the program within a derivation, in those cases, searching the wrapper derivation and other generated files and their contents is also acceptable. +If you are writing tests for a wrapper module, it is important to pass the name +of the wrapper to the first argument of the `test` function like in the example +below (marked at `(*)`). By doing this, we can grab the specified `wrapper.meta.platforms` config +of the wrapper (if any) and ensure that the tests are only run on the required platforms. -Example: ```nix { pkgs, - runCommand, self, + tlib, + ... }: + let - gitWrapped = self.wrappers.git.wrap { - inherit pkgs; - settings = { - user = { - name = "Test User"; - email = "test@example.com"; + inherit (tlib) + fileContains + isDirectory + isFile + notIsFile + areEqual + test + ; +in +test { wrapper = "direnv"; } { # <-- Specify the name of the wrapper here (*) + + "direnv wrapper should be created" = + let + wrapper = self.wrappers.direnv.wrap { + inherit pkgs; + nix-direnv.enable = true; }; - }; + in + [ + "[[ -d ${wrapper} ]]" # <-- a simple condition to be asserted + { + cond = "[[ -d ${wrapper} ]]"; + msg = "No directory found for wrapper."; # <-- you can also specify a custom error message + } + (isDirectory wrapper) # <-- or use pre-defined helpers + ]; + + "wrapper should output correct version" = + let + wrapper = self.wrappers.direnv.wrap { + inherit pkgs; + }; + in + '' # <-- no need to provide a list if there is only one assertion + "${wrapper}/bin/direnv" --version | + grep -q "${wrapper.version}" + ''; + + "math-tests" = { # <-- tests can be arbitrarily grouped + addition = [ + (areEqual 2 (1 + 1)) + (areEqual 7 (5 + 2)) + ]; + multiplication = [ + (areEqual 1 (1 * 1)) + (areEqual 10 (5 * 2)) + ]; }; -in -runCommand "git-test" { } '' - "${gitWrapped}/bin/git" config user.name | grep -q "Test User" - "${gitWrapped}/bin/git" config user.email | grep -q "test@example.com" - touch $out -'' +} ``` -If your module declares a list of valid platforms via its `meta.platforms` option, you should disable your test on the relevant platforms like so: + +Pre-defined assertions like `isDirectory` or `areEqual` are already available in tlib. +Feel free to contribute more if you find new ones that other maintainers might benefit from. + +### Writing Tests for Helper Modules or Library Functions + +The syntax is identical to [the example above](#writing-tests-for-wrappers), +except you don't provide a wrapper but a name: ```nix -if builtins.elem pkgs.stdenv.hostPlatform.system self.wrappers.waybar.meta.platforms then - pkgs.runCommand "waybar-test" { } '' - "${waybarWrapped}/bin/waybar" --version | grep -q "${waybarWrapped.version}" - touch $out - '' -else - null +{ + pkgs, + self, + tlib, + ... +}: + +let + inherit (tlib) + fileContains + isDirectory + isFile + notIsFile + areEqual + test + ; +in +test "my-test" { # <-- Specify an arbitrary name for your test +# test { name = "my-test" } { # <-- This is equivalent + + "my first test" = [ ... ]; # <-- nothing new here + "my second test" = [ ... ]; +} ``` If you are writing a helper module, or something very complex, you may wish to have multiple derivations. Simply return a set of them instead. -# Commit Messages +## Commit Messages Changes to wrapper modules should be titled `(wrapperModules.): some description`. For new additions, the description should be `init`, with any further explanation on subsequent lines @@ -212,6 +280,6 @@ For everything else, do the best you can to follow conventional commit message s Why specify this? I was having trouble figuring out what to title my commits. So now I know. -# Questions? +## Questions? The [github discussions board](https://github.com/BirdeeHub/nix-wrapper-modules/discussions) is open and a great place to find help! diff --git a/ci/docs/default.nix b/ci/docs/default.nix index e39ecc21..544655bd 100644 --- a/ci/docs/default.nix +++ b/ci/docs/default.nix @@ -208,6 +208,23 @@ in src = "${placeholder "generated"}/wrapper_docs/${n}.md"; }) (builtins.attrNames config.drv.wrapper_docs); } + { + name = "Contributing"; + data = "numbered"; + path = "md/CONTRIBUTING.md"; + src = ../../CONTRIBUTING.md; + subchapters = [ + { + name = "tlib"; + data = "numbered"; + path = "tlib.md"; + src = "${placeholder "out"}/wrappers-lib/tlib.md"; + build = '' + ${pkgs.nixdoc}/bin/nixdoc --category "" --description 'Testing library `tlib` documentation' --file '${../test-lib.nix}' --prefix "tlib" >> $out/wrappers-lib/tlib.md + ''; + } + ]; + } ]; }; } diff --git a/ci/flake.nix b/ci/flake.nix index d5740ac3..5d6e3965 100644 --- a/ci/flake.nix +++ b/ci/flake.nix @@ -20,6 +20,7 @@ inherit system; config.allowUnfree = true; }; + tlib = pkgs.callPackage ./test-lib.nix { inherit self; }; # Load checks from ci/checks/ directory coreAndCiChecks = lib.pipe ./checks [ @@ -43,7 +44,7 @@ name = "${prefix}-${name}"; inherit value; }; - result = pkgs.callPackage value { inherit self; }; + result = pkgs.callPackage value { inherit self tlib; }; in if result == null then [ ] diff --git a/ci/test-lib.nix b/ci/test-lib.nix new file mode 100644 index 00000000..297ea403 --- /dev/null +++ b/ci/test-lib.nix @@ -0,0 +1,328 @@ +{ + self, + lib, + runCommand, + writeShellScript, + stdenv, + pkgs, + ... +}: +let + wlib = self.lib; + + errMsg = msg: "(echo ${renderMsg msg} >&2 && return 1)"; + + indentBlock = + str: num: + let + idnt = " "; + lines = lib.splitString "\n" str; + indentLine = line: (wlib.repeatStr idnt num) + line; + in + lib.concatMapStringsSep "\n" indentLine lines; + + # Use [ANSI-C Quoting](https://www.gnu.org/software/bash/manual/html_node/ANSI_002dC-Quoting.html#ANSI_002dC-Quoting-1) + # to properly display multi-line messages without being indented if occurring in deeply nested tests. + renderMsg = + str: + let + lines = lib.splitString "\n" str; + result = "$'${lib.concatStringsSep "\\n" lines}'"; + in + if (lib.length lines) == 1 then str else result; + + toSanitizedJSON = + value: + if builtins.isAttrs value then + builtins.toJSON ( + lib.mapAttrsRecursive ( + path: v: + if builtins.isFunction v then + let + res = builtins.unsafeGetAttrPos (lib.last path) ( + lib.getAttrFromPath (lib.sublist 0 (builtins.length path - 1) path) value + ); + in + "" + else + v + ) value + ) + else + builtins.toJSON value; + + render = + node: + let + renderAssertion = + let + _renderAssertion = + { cond, msg }: + '' + ( + ${cond} + ) || ${errMsg msg}''; + in + assertion: + if lib.isString assertion then + _renderAssertion { + cond = assertion; + msg = "Failed assertion: ${assertion}"; + } + else + _renderAssertion assertion; + + renderAssertions = + assertions: lib.concatMapStringsSep " &&\n" (a: "(${renderAssertion a})") assertions; + + renderNode = + node: + let + block = + if !(lib.isAttrs node) then + renderAssertions (lib.toList node) + else + lib.concatMapAttrsStringSep " &&\n" (name: childNode: '' + (( + ${renderNode childNode} + ) || ${errMsg name})'') node; + in + indentBlock block 1; + in + renderNode node; + + _test = + { + name, + debug ? false, + }: + testSet: + let + testScript = '' + run () { + ${render testSet}; + } + run || (echo "Test '${name}' failed" >&2 && exit 1) + ''; + in + if debug then + writeShellScript name '' + ${testScript} + '' + else + runCommand name { passthru.test = testScript; } '' + ${testScript} + touch $out + ''; + +in +{ + /** + Takes tests provided as an attrs in `testSet`, renders it to a bash script, + and executes the rendered script using `pkgs.runCommand`. + + If the command runs successfully, the test set passes. Otherwise, a detailed + error message describing the failed assertion is shown in the logs. + + # Type + ``` + test :: (String | AttrSet) -> TestSet -> Derivation + ``` + + where: + + ``` + TestSet :: attrsOf (TestSet | Test) + + Test :: [ Assertion ] + + Assertion :: String | { cond :: String; msg :: String; } + ``` + + # Arguments + + settings + : Either a string or an attrs. + + If it is a string, it will be taken as the name of the derivation built + by `runCommand`. + + If it is a set, at least one of the keys `name` or `wrapper` need to be set + with a string value. + + - If you are defining tests for wrappers you should set `settings.wrapper` to the name + of the wrapper. This way, the `wrapper.meta.platforms` is considered during test execution, + and the test will only be run if the system it is running on is supported. + - If `settings.wrapper` is not set, the test will always be run. This makes sense if you + are testing core options or library functions. + - If only `settings.wrapper` is set, the name will be derived from this value by suffixing it with `-test`. + - If `settings.name` is set, it will be taken as the name of the derivation. + - If both are set, `settings.name` takes precedence. + + There is also an optional boolean option `settings.debug` (default = `false`). + If it is set to `true`, the generated bash script is built as an executable script that can be inspected + and run in `result` when running a specific test. + + The test can be disabled by setting `settings.enable = false`. + + testSet + : The set of tests to run. + */ + test = + settings: testSet: + let + name = + if lib.isString settings then + settings + else if settings ? name then + settings.name + else if settings ? wrapper then + "${settings.wrapper}-test" + else + throw '' + Invalid argument for `test`. + The first argument must be either a string (the test name) or an attrs + with at least one of the keys 'name' or 'wrapper', but got: + + ${lib.toSanitizedJSON settings} + ''; + wrapperName = settings.wrapper or null; + platforms = + if wrapperName == null then + null + else if (wrapperName != null) && (lib.hasAttr wrapperName self.wrappers) then + self.wrappers.${wrapperName}.meta.platforms + else + throw '' + Invalid argument for `test`. + The provided wrapper '${wrapperName}' was not found. + Available wrappers are: + + ${lib.toSanitizedJSON (lib.attrNames self.wrappers)} + ''; + enabled = + (settings.enable or true) + && (platforms == null || (platforms != null && builtins.elem stdenv.hostPlatform.system platforms)); + in + if enabled then + _test { + inherit name; + debug = settings.debug or false; + } testSet + else + null; + + /** + Returns an `Assertion` that checks whether `path` is an existing directory. + + # Type + ``` + isDirectory :: String -> Assertion + ``` + + # Arguments + path + : The filesystem path to check. + */ + isDirectory = path: { + cond = ''[[ -d "${path}" ]]''; + msg = "No such directory ${path}"; + }; + + /** + Returns an `Assertion` that checks whether `path` is an existing regular file. + + # Type + ``` + isFile :: String -> Assertion + ``` + + # Arguments + path + : The filesystem path to check. + */ + isFile = path: { + cond = ''[ -f "${path}" ]''; + msg = "No such file ${path}"; + }; + + /** + Returns an `Assertion` that checks whether `path` does **not** exist as a regular file. + + # Type + ``` + notIsFile :: String -> Assertion + ``` + + # Arguments + path + : The filesystem path that should be absent. + */ + notIsFile = path: { + cond = ''[ ! -f "${path}" ]''; + msg = "File ${path} should not exist"; + }; + + /** + Returns an `Assertion` that checks whether `file` contains a line matching `pattern`. + + The check is performed with `grep -q`, so `pattern` is treated as a basic regular expression. + + # Type + ``` + fileContains :: String -> String -> Assertion + ``` + + # Arguments + file + : Path to the file to search. + + pattern + : Basic regular expression to search for. + */ + fileContains = file: pattern: { + cond = ''grep -q '${pattern}' "${file}"''; + msg = "Pattern '${pattern}' not found in ${file}"; + }; + + /** + Returns an `Assertion` that checks whether `expected` and `actual` are equal. + + The comparison is performed in Nix at evaluation time. If the values differ, + the assertion message shows both values serialised as JSON. + + # Type + ``` + areEqual :: Any -> Any -> Assertion + ``` + + # Arguments + expected + : The expected value. + + actual + : The value to compare against `expected`. + */ + areEqual = + expected: actual: + let + equal = expected == actual; + in + { + cond = if equal then "true" else "false"; + msg = + if !equal then + '' + Expected: + ${toSanitizedJSON expected} + but got: + ${toSanitizedJSON actual}'' + else + "This should never happen."; + }; +} diff --git a/maintainers/default.nix b/maintainers/default.nix index b6bd9481..9d3b2773 100644 --- a/maintainers/default.nix +++ b/maintainers/default.nix @@ -61,6 +61,11 @@ githubId = 8916363; name = "Nikita Wootten"; }; + zenoli = { + name = "Zenoli"; + github = "zenoli"; + githubId = 8073528; + }; pengolord = { name = "pengo"; email = "pbalternates@gmail.com";