From 099dc0c1be9eaaa29863d0d4e0a6aeb5e6f37189 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:50:46 +0000 Subject: [PATCH 01/38] build(nix): introduce nix build system The goal of this series of commits is to migrate functionality from dpdk-sys (and give it a much needed refactor in the process). More specifically, this series of commits is intended to get the project to the point where you can do a sterile build of DPDK with the correct set of CFLAGS/LDFLAGS in both the debug and release profiles. Forming that build into a sysroot which can actually replace dpdk-sys will be left as future work. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 1 + 1 file changed, 1 insertion(+) create mode 100644 nix/overlays/dataplane.nix diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix new file mode 100644 index 000000000..307698934 --- /dev/null +++ b/nix/overlays/dataplane.nix @@ -0,0 +1 @@ +{}: final: prev: {} From 116b04eee5e01d1375e5ac5c75e188fd026d84a6 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:52:02 +0000 Subject: [PATCH 02/38] build(nix): intro addToEnv helper We begin the construction of our sysroot with a helper function designed to inject environment variables into a nixpkgs provided stdenv. This will (eventually) allow us to customize the flags used to compile each package needed for our sysroot. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 307698934..7d4670ae4 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -1 +1,11 @@ -{}: final: prev: {} +{ }: +final: prev: +let + helpers.addToEnv = + add: orig: + orig + // ( + with builtins; (mapAttrs (var: val: (toString (orig.${var} or "")) + " " + (toString val)) add) + ); +in +{ } From 52f6352512fcba9fc5e02c4731ef5bc17636988f Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:53:06 +0000 Subject: [PATCH 03/38] build(nix): introduce stdenv-llvm This commit introduces a nixpkgs provided stdenv based on the LLVM toolchain. We explicitly forswear GCC for this sysroot on the grounds that rustc is based on LLVM. Given that we want LTO to work properly, we need to ensure that all of our (modest list of) dataplane dependencies are built with LLVM. In a future commit we will add infrastructure to ensure that we are also always using the same version of LLVM used by our selected version of rustc. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 7d4670ae4..9559ec053 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -7,5 +7,7 @@ let // ( with builtins; (mapAttrs (var: val: (toString (orig.${var} or "")) + " " + (toString val)) add) ); + adapt = final.stdenvAdapters; + stdenv-llvm = adapt.makeStaticLibraries final.buildPackages.llvmPackages.stdenv; in { } From 8bcafb7d2b4fabd51bfa89424b7f9c17a043ddd7 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:56:10 +0000 Subject: [PATCH 04/38] build(nix): add llvm's bintools and lld to stdenv For reasons which make no sense to me, llvmPackages.stdenv does not seem to add LLVM's bintools or linker to the nativeBuildInputs by default. If you attempt to pass `-fuse-ld=lld` to the compiler without explicitly adding them to the build time dependency list then you fail to link. The fix is easy: inject those packages as dependencies. Note that I deliberately don't use `pkgs.lld` as that version of lld is not wrapped by nix and will not ensure correct settings for the rpath in generated elf files. llvmPackages.lld is the version we need here, as that version of lld is wrapped. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 9559ec053..aa315257b 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -8,6 +8,13 @@ let with builtins; (mapAttrs (var: val: (toString (orig.${var} or "")) + " " + (toString val)) add) ); adapt = final.stdenvAdapters; - stdenv-llvm = adapt.makeStaticLibraries final.buildPackages.llvmPackages.stdenv; + bintools = final.buildPackages.llvmPackages.bintools; + lld = final.buildPackages.llvmPackages.lld; + stdenv-llvm = adapt.addAttrsToDerivation (orig: { + nativeBuildInputs = (orig.nativeBuildInputs or [ ]) ++ [ + bintools + lld + ]; + }) (adapt.makeStaticLibraries final.buildPackages.llvmPackages.stdenv); in { } From 9ef23ba019afc91d7ce65577aa1da62710d91e7e Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:56:43 +0000 Subject: [PATCH 05/38] build(nix): disable checks in stdenv It is neat that nixpkgs attempts to run unit tests for compiled packages, but in my experience those tests spuriously fail on the regular. The cause of these failures is almost uniformly due to insufficient permissions in the build environment. I will leave the testing of packages to the developers and package maintainers and simply disable those tests in our overlay. This is especially important in that we will (in the future) cross compile for aarch64 and our build machine would be unable to run the tests anyway. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index aa315257b..ccc73195d 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -11,6 +11,7 @@ let bintools = final.buildPackages.llvmPackages.bintools; lld = final.buildPackages.llvmPackages.lld; stdenv-llvm = adapt.addAttrsToDerivation (orig: { + doCheck = false; nativeBuildInputs = (orig.nativeBuildInputs or [ ]) ++ [ bintools lld From 01a9d6b123f61a49d0d1878ff96d809ba97a666c Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:57:47 +0000 Subject: [PATCH 06/38] build(nix): intro stdenv-llvm-with-flags Invoke or previously built helper function to pass user supplied flags into our new llvm stdenv. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index ccc73195d..04debf274 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -1,4 +1,6 @@ -{ }: +{ + env ? { }, +}: final: prev: let helpers.addToEnv = @@ -17,5 +19,8 @@ let lld ]; }) (adapt.makeStaticLibraries final.buildPackages.llvmPackages.stdenv); + stdenv-llvm-with-flags = adapt.addAttrsToDerivation (orig: { + env = helpers.addToEnv env (orig.env or { }); + }) stdenv-llvm; in { } From ea6a91eb6c0c49a953de06f2c046c4020726c9ca Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:58:18 +0000 Subject: [PATCH 07/38] build(nix): intro dataplane-dep helper function Cook a quick helper function to override the stdenv of select packages with our static + llvm + user supplied flags stdenv. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 04debf274..8864d88f2 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -22,5 +22,6 @@ let stdenv-llvm-with-flags = adapt.addAttrsToDerivation (orig: { env = helpers.addToEnv env (orig.env or { }); }) stdenv-llvm; + dataplane-dep = pkg: pkg.override { stdenv = stdenv-llvm-with-flags; }; in { } From 177cfbb7af9767539799fe2ba0f8b07ca2957ce5 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:58:59 +0000 Subject: [PATCH 08/38] build(nix): null ethtool and iproute2 See comment for my reasoning. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 8864d88f2..e7d0d9d9c 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -24,4 +24,14 @@ let }) stdenv-llvm; dataplane-dep = pkg: pkg.override { stdenv = stdenv-llvm-with-flags; }; in -{ } +{ + # Don't bother adapting ethtool or iproute2's build to our custom flags / env. Failure to null this can trigger + # _massive_ builds because ethtool depends on libnl (et al), and we _do_ overlay libnl. Thus, the ethtool / iproute2 + # get rebuilt and you end up rebuilding the whole world. + # + # To be clear, we can still use ethtool / iproute2 if we want, we just don't need to optimize / lto it. + # If you want to include ethtool / iproute2, I recommend just cutting another small overlay and static linking them. + # Alternatively, you could skip that and just ship the default build of ethtool. + ethtool = null; + iproute2 = null; +} From e975328ea4226199b12fbeefc9b7686a3347a277 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:59:24 +0000 Subject: [PATCH 09/38] build(nix): null heavy doc tools See comment for my reasoning. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index e7d0d9d9c..476fdc207 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -34,4 +34,12 @@ in # Alternatively, you could skip that and just ship the default build of ethtool. ethtool = null; iproute2 = null; + + # These are only used in docs and can make our build explode in size if we let any of this rebuild in this overlay. + # It is much easier to just not build docs in this overlay. We don't care if the build depends on pandoc per se, but + # you will regret the need to rebuild ghc :shrug: + gd = null; + graphviz = null; + mscgen = null; + pandoc = null; } From 7b5e4c7fa4a0da9713d3ea4408d6b1b716c31f6a Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 00:59:57 +0000 Subject: [PATCH 10/38] build(nix): null systemd + helpers See comment for my reasoning. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 476fdc207..7527d2e20 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -42,4 +42,15 @@ in graphviz = null; mscgen = null; pandoc = null; + + # We should avoid accepting anything in our dpdk + friends pkgs which depends on udev / systemd; our deploy won't + # support any such mechanisms. + # + # Usually this type of dependency takes the form of udev rules / systemd service files being generated (which is no + # problem). That said, builds which hard and fast depend on systemd or udev are very suspicious in this context, so + # exceptions to this removal should be granted with care and some level of prejudice. At minimum, such exceptions + # tend to make it hard to cross compile which is an important test case for our sysroot. + systemd = null; + udev = null; + udevCheckHook = null; } From 5c60d4d8e6aa63d4c32000c97d87dbe1bf0b8658 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:00:41 +0000 Subject: [PATCH 11/38] build(nix): introduce overlay for libmd This is an indirect dependency of DPDK, so we want to customize its build. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 7527d2e20..b2ff0274f 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -53,4 +53,24 @@ in systemd = null; udev = null; udevCheckHook = null; + + # libmd is used by libbsd (et al) which is an optional dependency of dpdk. + # + # We _might_ actually care about perf here, so we lto this package. + # At minimum, the provided functions are generally quite small and likely to benefit from inlining, so static linking + # is a solid plan. + libmd = (dataplane-dep prev.libmd).overrideAttrs (orig: { + outputs = (orig.outputs or [ "out" ]) ++ [ "static" ]; + # we need to enable shared libs (in addition to static) to make dpdk's build happy. Basically, DPDK's build has no + # means of disabling shared libraries, and it doesn't really make any sense to static link this into each .so + # file. Ideally we would just _not_ build those .so files, but that would require doing brain surgery on dpdk's + # meson build, and maintaining such a change set is not worth it to avoid building some .so files. + configureFlags = (orig.configureFlags or [ ]) ++ [ + "--enable-shared" + ]; + postInstall = (orig.postInstall or "") + '' + mkdir -p "$static/lib"; + mv $out/lib/*.a $static/lib; + ''; + }); } From 1185bca0df14eac1f4bcdb64203ebd6e98477ae7 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:01:05 +0000 Subject: [PATCH 12/38] build(nix): introduce overlay for libbsd This is an indirect dependency of DPDK, so we want to customize its build. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index b2ff0274f..725b9dab6 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -73,4 +73,22 @@ in mv $out/lib/*.a $static/lib; ''; }); + + # This is a (technically optional) dependency of DPDK used for secure string manipulation and some hashes we value; + # static link + lto for sure. + # + # This is also a reasonably important target for `-fsanitize=cfi` and or `-fsanitize=safe-stack` as libbsd provides + # more secure versions of classic C string manipulation utilities, and I'm all about that defense-in-depth. + libbsd = (dataplane-dep prev.libbsd).overrideAttrs (orig: { + outputs = (orig.outputs or [ "out" ]) ++ [ "static" ]; + # we need to enable shared (in addition to static) to build dpdk. + # See the note on libmd for reasoning. + configureFlags = orig.configureFlags ++ [ + "--enable-shared" + ]; + postInstall = (orig.postInstall or "") + '' + mkdir -p "$static/lib"; + mv $out/lib/*.a $static/lib; + ''; + }); } From 17cf416a9dbf4fecfa06489f4e90b9c4df60ca77 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:01:27 +0000 Subject: [PATCH 13/38] build(nix): introduce overlay for libnl This is an dependency of DPDK, so we want to customize its build. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 725b9dab6..f5140f384 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -91,4 +91,18 @@ in mv $out/lib/*.a $static/lib; ''; }); + + # This is (for better or worse) used by dpdk to parse / manipulate netlink messages. + # + # We don't care about performance here, so this may be a good candidate for size reduction compiler flags like -Os. + # + # That said, we don't currently have infrastructure to pass flags at a per package level and building that is more + # trouble than a minor reduction in binary size / instruction cache pressure is likely worth. Also, lto doesn't + # currently love size optimizations. The better option is likely to use PGO + BOLT to put these functions far away + # from the hot path in the final ELF file's layout and just ignore that this stuff is compiled with -O3 and friends. + # + # More, this is a very low level library designed to send messages between a privileged process and the kernel. + # The simple fact that this appears in our toolchain justifies sanitizers like safe-stack and cfi and/or flags like + # -fcf-protection=full. + libnl = dataplane-dep prev.libnl; } From e88dbb2fc42e1cd98c3a580545402383d1ea802d Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:01:49 +0000 Subject: [PATCH 14/38] build(nix): introduce overlay for numactl This is an dependency of DPDK, so we want to customize its build. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index f5140f384..3f162ff90 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -105,4 +105,30 @@ in # The simple fact that this appears in our toolchain justifies sanitizers like safe-stack and cfi and/or flags like # -fcf-protection=full. libnl = dataplane-dep prev.libnl; + + # This is needed by DPDK in order to determine which pinned core runs on which numa node and which NIC is most + # efficiently connected to which NUMA node. You can disable the need for this library entirely by editing dpdk's + # build to specify `-Dmax_numa_nodes=1`. + # + # While we don't currently hide NUMA mechanics from DPDK, there is something to be said for eliminating this library + # from our toolchain as a fair level of permissions and a lot of different low level trickery is required to make it + # function. In "the glorious future" we should bump all of this logic up to the dataplane's init process, compute + # what we need to, pre-mmap _all_ of our heap memory, configure our cgroups and CPU affinities, and then pin our cores + # and use memory pools local to the numa node of the pinned core. That would be a fair amount of work, but it would + # liminate a fairly large dependency and likely increase the performance and security of the dataplane. + # + # For now, we leave this on so DPDK can do some of that for us. That said, this logic is quite cold and would ideally + # be size optimized and punted far from all hot paths. BOLT should be helpful here. + numactl = (dataplane-dep prev.numactl).overrideAttrs (orig: { + outputs = (prev.lib.lists.remove "man" orig.outputs) ++ [ "static" ]; + # we need to enable shared (in addition to static) to build dpdk. + # See the note on libmd for reasoning. + configureFlags = (orig.configureFlags or [ ]) ++ [ + "--enable-shared" # dpdk does not like to build its .so files if we don't build numa.so as well + ]; + postInstall = (orig.postInstall or "") + '' + mkdir -p "$static/lib"; + mv $out/lib/*.a $static/lib; + ''; + }); } From e89fc4ab601bd5b96b8252424cd696ac653fe91b Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:02:36 +0000 Subject: [PATCH 15/38] build(nix): introduce overlay for rdma-core This is an dependency of DPDK, so we want to customize its build. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 3f162ff90..95a01bc8c 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -131,4 +131,36 @@ in mv $out/lib/*.a $static/lib; ''; }); + + # This is one of the two most important to optimize components of the whole build (along with dpdk itself). + # + # RDMA-core is the low level building block for many of the PMDs within DPDK including the mlx5 PMD. It is a + # performance and security critical library which we will likely never be able to remove from our dependencies. + # + # Some of this library is almost always called in a very tight loop, especially as used by DPDK PMDs. It is happy to + # link dynamically or statically, and we should make a strong effort to make sure that we always pick static linking + # to enable inlining (wherever the compiler decides it makes sense). You very likely want to enable lto here in any + # release build. + rdma-core = (dataplane-dep prev.rdma-core).overrideAttrs (orig: { + outputs = [ + "dev" + "out" + "static" + ]; + cmakeFlags = orig.cmakeFlags ++ [ + "-DENABLE_STATIC=1" + # we don't need pyverbs, and turning it off reduces build time / complexity. + "-DNO_PYVERBS=1" + # no need for docs in container images. + "-DNO_MAN_PAGES=1" + # we don't care about this lib's exported symbols / compat situation _at all_ because we static link (which + # doesn't even have symbol versioning / compatibility in the first place). Turning this off just reduces the + # build's internal complexity and makes lto easier. + "-DNO_COMPAT_SYMS=1" + ]; + postInstall = (orig.postInstall or "") + '' + mkdir -p $static/lib; + mv $out/lib/*.a $static/lib/ + ''; + }); } From 3ae374099f00c93ba59d52dcb279c3c91013ae30 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:05:52 +0000 Subject: [PATCH 16/38] build(nix): introduce npins package management This is a handy tool for "pinning" our external dependencies to known versions with fixed cryptographic hashes. I tried to make this work with nix flakes, but I had an extremely difficult time making cross compile work, and I frankly don't understand what flakes does that npins doesn't do objectively better. Signed-off-by: Daniel Noland --- npins/default.nix | 146 +++++++++++++++++++++++++++++++++++++++++++++ npins/sources.json | 11 ++++ shell.nix | 1 + 3 files changed, 158 insertions(+) create mode 100644 npins/default.nix create mode 100644 npins/sources.json diff --git a/npins/default.nix b/npins/default.nix new file mode 100644 index 000000000..659247626 --- /dev/null +++ b/npins/default.nix @@ -0,0 +1,146 @@ +/* + This file is provided under the MIT licence: + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +# Generated by npins. Do not modify; will be overwritten regularly +let + data = builtins.fromJSON (builtins.readFile ./sources.json); + version = data.version; + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = + first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # If the environment variable NPINS_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + # (Taken from Niv for compatibility) + mayOverride = + name: path: + let + envVarName = "NPINS_OVERRIDE_${saneName}"; + saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name; + ersatz = builtins.getEnv envVarName; + in + if ersatz == "" then + path + else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + builtins.trace "Overriding path of \"${name}\" with \"${ersatz}\" due to set \"${envVarName}\"" ( + if builtins.substring 0 1 ersatz == "/" then + /. + ersatz + else + /. + builtins.getEnv "PWD" + "/${ersatz}" + ); + + mkSource = + name: spec: + assert spec ? type; + let + path = + if spec.type == "Git" then + mkGitSource spec + else if spec.type == "GitRelease" then + mkGitSource spec + else if spec.type == "PyPi" then + mkPyPiSource spec + else if spec.type == "Channel" then + mkChannelSource spec + else if spec.type == "Tarball" then + mkTarballSource spec + else + builtins.throw "Unknown source type ${spec.type}"; + in + spec // { outPath = mayOverride name path; }; + + mkGitSource = + { + repository, + revision, + url ? null, + submodules, + hash, + branch ? null, + ... + }: + assert repository ? type; + # At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository + # In the latter case, there we will always be an url to the tarball + if url != null && !submodules then + builtins.fetchTarball { + inherit url; + sha256 = hash; # FIXME: check nix version & use SRI hashes + } + else + let + url = + if repository.type == "Git" then + repository.url + else if repository.type == "GitHub" then + "https://github.com/${repository.owner}/${repository.repo}.git" + else if repository.type == "GitLab" then + "${repository.server}/${repository.repo_path}.git" + else + throw "Unrecognized repository type ${repository.type}"; + urlToName = + url: rev: + let + matched = builtins.match "^.*/([^/]*)(\\.git)?$" url; + + short = builtins.substring 0 7 rev; + + appendShort = if (builtins.match "[a-f0-9]*" rev) != null then "-${short}" else ""; + in + "${if matched == null then "source" else builtins.head matched}${appendShort}"; + name = urlToName url revision; + in + builtins.fetchGit { + rev = revision; + inherit name; + # hash = hash; + inherit url submodules; + }; + + mkPyPiSource = + { url, hash, ... }: + builtins.fetchurl { + inherit url; + sha256 = hash; + }; + + mkChannelSource = + { url, hash, ... }: + builtins.fetchTarball { + inherit url; + sha256 = hash; + }; + + mkTarballSource = + { + url, + locked_url ? url, + hash, + ... + }: + builtins.fetchTarball { + url = locked_url; + sha256 = hash; + }; +in +if version == 5 then + builtins.mapAttrs mkSource data.pins +else + throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`" diff --git a/npins/sources.json b/npins/sources.json new file mode 100644 index 000000000..ba6d59bfc --- /dev/null +++ b/npins/sources.json @@ -0,0 +1,11 @@ +{ + "pins": { + "nixpkgs": { + "type": "Channel", + "name": "nixpkgs-unstable", + "url": "https://releases.nixos.org/nixpkgs/nixpkgs-26.05pre911335.23735a82a828/nixexprs.tar.xz", + "hash": "03cv7yy3ldb3i50in6qkm98y68nlid874l52wayzgx0z7pbpq1rk" + } + }, + "version": 5 +} diff --git a/shell.nix b/shell.nix index 112c4b8a4..918e6cad1 100644 --- a/shell.nix +++ b/shell.nix @@ -12,6 +12,7 @@ just nil nixd + npins wget ]); }).env From b1364208443812b335136be85a952684b5965ce0 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:07:22 +0000 Subject: [PATCH 17/38] build(nix): pin our version of rdma-core We want to have exact control over our version of rdma-core. We pin the version to use our fork of rdma-core which contains a minor patch sequence to help LTO compiles. This pin was added via ``` npins add github githedgehog rdma-core -b fix-lto-60.0 ``` Signed-off-by: Daniel Noland --- npins/sources.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/npins/sources.json b/npins/sources.json index ba6d59bfc..78425d3b3 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -5,6 +5,19 @@ "name": "nixpkgs-unstable", "url": "https://releases.nixos.org/nixpkgs/nixpkgs-26.05pre911335.23735a82a828/nixexprs.tar.xz", "hash": "03cv7yy3ldb3i50in6qkm98y68nlid874l52wayzgx0z7pbpq1rk" + }, + "rdma-core": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "githedgehog", + "repo": "rdma-core" + }, + "branch": "fix-lto-60.0", + "submodules": false, + "revision": "9ae1b26593e2cb53239e1124f88ce1698d53857e", + "url": "https://github.com/githedgehog/rdma-core/archive/9ae1b26593e2cb53239e1124f88ce1698d53857e.tar.gz", + "hash": "1djdsfga9if02pl8ynnyyf640xdd37fha6zp3xlylciy8apzn1r4" } }, "version": 5 From e161db519db2fd6c3bb24cd65b5d878fd0fb315e Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:18:04 +0000 Subject: [PATCH 18/38] build(nix): import pinned sources into default.nix Now that we have our pinned version of nixpkgs and rdma-core set up we can pipe those sources into our build system for consumption by the rest of our overlay. Signed-off-by: Daniel Noland --- default.nix | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 default.nix diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..a04a84bb5 --- /dev/null +++ b/default.nix @@ -0,0 +1,9 @@ +let + sources = import ./npins; +in +{ + +}: +{ + inherit sources; +} From ab64fcbaa7bd7d87fa6f0d81ffeda6f56d918ce1 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:18:20 +0000 Subject: [PATCH 19/38] build(nix): import nixpkgs from npins source Use the pinned version of nixpkgs. Signed-off-by: Daniel Noland --- default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/default.nix b/default.nix index a04a84bb5..3933bb4ce 100644 --- a/default.nix +++ b/default.nix @@ -1,5 +1,6 @@ let sources = import ./npins; + pkgs = import sources.nixpkgs { }; in { From 2ee1e8467c97e8203fb7b9bd90cdafb6f00979a8 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:20:18 +0000 Subject: [PATCH 20/38] build(nix): introduce default overlay We currently only have one overlay in play, but I expect that to change when I add additional logic for FRR. Regardless, there is little cost in making the set of selected overlays flexible. Signed-off-by: Daniel Noland --- nix/overlays/default.nix | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 nix/overlays/default.nix diff --git a/nix/overlays/default.nix b/nix/overlays/default.nix new file mode 100644 index 000000000..7f827c35b --- /dev/null +++ b/nix/overlays/default.nix @@ -0,0 +1,4 @@ +{ }: +{ + dataplane = import ./dataplane.nix { env = { }; }; +} From 8d3cb34809e7148b179906462df5460ef83e3568 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:22:08 +0000 Subject: [PATCH 21/38] build(nix): pipe pinned sources to dataplane overlay These pinned sources need to be consumed by the dataplane overlay in order to pin our external dependencies, rdma-core and (soon) dpdk. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 1 + nix/overlays/default.nix | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 95a01bc8c..5b7b8ab76 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -1,4 +1,5 @@ { + sources, env ? { }, }: final: prev: diff --git a/nix/overlays/default.nix b/nix/overlays/default.nix index 7f827c35b..f112df68c 100644 --- a/nix/overlays/default.nix +++ b/nix/overlays/default.nix @@ -1,4 +1,9 @@ -{ }: { - dataplane = import ./dataplane.nix { env = { }; }; + sources, +}: +{ + dataplane = import ./dataplane.nix { + inherit sources; + env = { }; + }; } From 807a8d2fe13c48e3e4ef7377b3d870f67b72bdcd Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:22:58 +0000 Subject: [PATCH 22/38] build(nix): use pinned rdma-core Use the newly piped in value of sources to fix rdma-core to our chosen version. Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 5b7b8ab76..624e1ed36 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -143,6 +143,8 @@ in # to enable inlining (wherever the compiler decides it makes sense). You very likely want to enable lto here in any # release build. rdma-core = (dataplane-dep prev.rdma-core).overrideAttrs (orig: { + version = sources.rdma-core.branch; + src = sources.rdma-core.outPath; outputs = [ "dev" "out" From d51d22e042bbc88b13fd31f70db4e725a77fc42b Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:08:56 +0000 Subject: [PATCH 23/38] build(nix): pin DPDK to version 25.11 DPDK recently released a new LTS version (25.11). Given that we are reworking our build system anyway, there is little reason to stay on the (now) outdated 25.07 version. This process will (in a future commit series) require regenerating our rust bindings anyway. While that is a relatively easy task, I can't see any real reason to do it twice. This pin was generated with the command ``` npins add github DPDK dpdk -b 25.11 ``` Signed-off-by: Daniel Noland --- npins/sources.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/npins/sources.json b/npins/sources.json index 78425d3b3..bc55da807 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -1,5 +1,18 @@ { "pins": { + "dpdk": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "DPDK", + "repo": "dpdk" + }, + "branch": "25.11", + "submodules": false, + "revision": "ed957165eadbe60a47d5ec223578cdd1c13d0bd9", + "url": "https://github.com/DPDK/dpdk/archive/ed957165eadbe60a47d5ec223578cdd1c13d0bd9.tar.gz", + "hash": "09h7wnmq4c9xm1nsyv5mz1yf91c1l6vy9sdcamb09qjjx4wgs0q9" + }, "nixpkgs": { "type": "Channel", "name": "nixpkgs-unstable", From b394208be9363400edadf2a1557eec4e30a3e7b2 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:27:49 +0000 Subject: [PATCH 24/38] build(nix): introduce dpdk package This commit introduces a build recipe for DPDK optimized for our use case. This is largely just transferred over from dpdk-sys. The primary reason to focus on minimizing DPDK is to make it as easy as possible to bind DPDK to rust. In particular, enabling all of the drivers significantly complicates the binding process and significantly increases build times. Signed-off-by: Daniel Noland --- nix/pkgs/dpdk/default.nix | 284 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 nix/pkgs/dpdk/default.nix diff --git a/nix/pkgs/dpdk/default.nix b/nix/pkgs/dpdk/default.nix new file mode 100644 index 000000000..44d4474bd --- /dev/null +++ b/nix/pkgs/dpdk/default.nix @@ -0,0 +1,284 @@ +{ + src, + stdenv, + lib, + pkg-config, + meson, + ninja, + libbsd, + numactl, + rdma-core, + libnl, + python3, + build-params ? { + lto = "true"; + build-type = "release"; # "debug" | "release" + }, +}: + +stdenv.mkDerivation { + pname = "dpdk"; + version = src.branch; + src = src.outPath; + nativeBuildInputs = [ + meson + ninja + pkg-config + python3 + python3.pkgs.pyelftools + ]; + + buildInputs = [ + libbsd + libnl + numactl + rdma-core + ]; + + postPatch = '' + patchShebangs config/arm buildtools + # We have no use for RTE_TRACE at all and it makes things more difficult from a security POV so disable it + sed -i 's/#define RTE_TRACE 1/#undef RTE_TRACE/g' config/rte_config.h + # We have no use for receive or transmit callbacks at this time so disable them + sed -i 's/#define RTE_ETHDEV_RXTX_CALLBACKS 1/#undef RTE_ETHDEV_RXTX_CALLBACKS/g' config/rte_config.h + ''; + + mesonFlags = + let + disabledLibs = [ + "acl" + "argparse" + "bbdev" + "bitratestats" + "bpf" + "cfgfile" + "compressdev" + "dispatcher" + "distributor" + "efd" + "fib" + "gpudev" + "graph" + "gro" + "gso" + "ip_frag" + "ipsec" + "jobstats" + "latencystats" + "lpm" + "member" + "metrics" + "mldev" + "node" + "pcapng" + "pdcp" + "pdump" + "pipeline" + "port" + "power" + "ptr_compress" + "rawdev" + "regexdev" + "reorder" + "rib" + "sched" + "table" + ]; + enabledLibs = [ + "cryptodev" # required for vhost + "dmadev" # required by vhost + "ethdev" + "eventdev" + "pci" + "security" + "timer" + "vhost" + ]; + disabledDrivers = [ + "baseband/*" + "bus/ifpga" + "bus/vdev" + "bus/vmbus" + "common/cnxk" + "common/cpt" + "common/dpaax" + "common/octeontx" + "common/octeontx2" + "common/qat" + "common/sfc_efx" + "compress/*" + "compress/mlx5" + "compress/zlib" + "crypto/*" + "crypto/aesni_gcm" + "crypto/aesni_mb" + "crypto/bcmfs" + "crypto/ccp" + "crypto/kasumi" + "crypto/mlx5" + "crypto/nitrox" + "crypto/null" + "crypto/openssl" + "crypto/scheduler" + "crypto/snow3g" + "crypto/virtio" + "crypto/zuc" + "event/dlb" + "event/dsw" + "event/opdl" + "event/skeleton" + "event/sw" + "net/acc100" + "net/af_packet" + "net/af_xdp" + "net/ark" + "net/atlantic" + "net/avp" + "net/axgbe" + "net/bcmfs" + "net/bnx2x" + "net/bnxt" + "net/bond" + "net/caam_jr" + "net/ccp" + "net/cnxk" + "net/cnxk_bphy" + "net/cpt" + "net/cxgbe" + "net/dlb2" + "net/dpaa" + "net/dpaa2" + "net/dpaa2_cmdif" + "net/dpaa2_qdma" + "net/dpaa2_sec" + "net/dpaa_sec" + "net/dpaax" + "net/dsw" + "net/ena" + "net/enetc" + "net/enic" + "net/failsafe" + "net/fm10k" + "net/fpga_5gnr_fec" + "net/fpga_lte_fec" + "net/fslmc" + "net/hinic" + "net/hns3" + "net/ifc" + "net/ifpga" + "net/igc" + "net/ioat" + "net/ionic" + "net/ipn3ke" + "net/kasumi" + "net/kni" + "net/liquidio" + "net/memif" + "net/mlx4" + "net/netvsc" + "net/nfp" + "net/ngbe" + "net/nitrox" + "net/ntb" + "net/null" + "net/octeontx" + "net/octeontx2" + "net/octeontx2_dma" + "net/octeontx2_ep" + "net/octeontx_ep" + "net/opdl" + "net/pcap" + "net/pfe" + "net/qede" + "net/sfc" + "net/sfc_efx" + "net/skeleton" + "net/snow3g" + "net/softnic" + "net/tap" + "net/thunderx" + "net/turbo_sw" + "net/txgbe" + "net/vdev" + "net/vdev_netvsc" + "net/vmbus" + "net/vmxnet3" + "net/zuc" + "raw/*" + "raw/ioat" + "raw/ntb" + "raw/skeleton" + "regex/*" + "regex/mlx5" + "vdpa/*" + "vdpa/ifc" + ]; + enabledDrivers = [ + "bus/auxiliary" + "bus/pci" + "common/mlx5" + "mempool/bucket" + "mempool/ring" + "mempool/stack" + "net/auxiliary" + "net/dmadev" + "net/intel/e1000" + "net/intel/i40e" + "net/intel/iavf" + "net/intel/ixgbe" + "net/mlx5" + "net/ring" + "net/vhost" + "net/virtio" + "vdpa/mlx5" + ]; + in + with build-params; + [ + "--buildtype=${build-type}" + "-Dauto_features=disabled" + "-Db_colorout=never" + "-Db_lto=${lto}" + "-Db_lundef=true" + "-Db_pgo=off" + "-Db_pie=true" + "-Dbackend=ninja" + "-Ddefault_library=static" + "-Denable_docs=false" + "-Denable_driver_sdk=false" + "-Dmax_numa_nodes=8" + "-Dtests=false" # Running DPDK tests in CI is usually silly + "-Duse_hpet=false" + "-Ddebug=false" + ''-Ddisable_drivers=${lib.concatStringsSep "," disabledDrivers}'' + ''-Denable_drivers=${lib.concatStringsSep "," enabledDrivers}'' + ''-Denable_libs=${lib.concatStringsSep "," enabledLibs}'' + ''-Ddisable_libs=${lib.concatStringsSep "," disabledLibs}'' + ]; + + outputs = [ + "out" + "static" + "dev" + "share" + ]; + + postInstall = '' + # Remove docs. We don't build these anyway + rm -rf $out/share/doc + mkdir -p $static/lib $share; + mv $out/lib/*.a $static/lib + mv $out/share $share + ''; + + meta = with lib; { + description = "Set of libraries and drivers for fast packet processing"; + homepage = "http://dpdk.org/"; + license = with licenses; [ + lgpl21 + gpl2 + bsd2 + ]; + platforms = platforms.linux; + }; +} From 8f8871f715be210cfaa822dbc853ff87b94ffeed Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:30:34 +0000 Subject: [PATCH 25/38] build(nix): replace dpdk build in dataplane overlay Call dpdk package from this repo rather than using the upstream build instructions (which are somewhat unfriendly to bindgen). Signed-off-by: Daniel Noland --- nix/overlays/dataplane.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nix/overlays/dataplane.nix b/nix/overlays/dataplane.nix index 624e1ed36..3487cba14 100644 --- a/nix/overlays/dataplane.nix +++ b/nix/overlays/dataplane.nix @@ -166,4 +166,14 @@ in mv $out/lib/*.a $static/lib/ ''; }); + + # Compiling DPDK is the primary objective of this overlay. + # + # We care _a lot_ about how this is compiled and should always use flags which are either optimized for performance + # or debugging. After all, if you aren't doing something performance critical then I don't know why you want DPDK at + # all :) + # + # Also, while this library has a respectable security track record, this is also a super strong candidate for + # cfi, safe-stack, and cf-protection. + dpdk = dataplane-dep (final.callPackage ../pkgs/dpdk { src = sources.dpdk; }); } From 5a5282f47b9c5230701debb740301b0fb1dd95bf Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:31:18 +0000 Subject: [PATCH 26/38] build(nix): expose pkgs in default.nix Now that our overlay is mostly ready, we can re-export our overlay plus nixpkgs from default.nix. Signed-off-by: Daniel Noland --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 3933bb4ce..b7c6f15e5 100644 --- a/default.nix +++ b/default.nix @@ -6,5 +6,5 @@ in }: { - inherit sources; + inherit sources pkgs; } From 3c6e09feae02bae077db94ec3b37b6cf68d51007 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:33:03 +0000 Subject: [PATCH 27/38] build(nix): add dataplane overlay to default.nix Now that our overlay is mostly cooked, we can inject it into our pinned version of nixpkgs. Signed-off-by: Daniel Noland --- default.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index b7c6f15e5..ec67ab546 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,13 @@ let sources = import ./npins; - pkgs = import sources.nixpkgs { }; + overlays = import ./nix/overlays { + inherit sources; + }; + pkgs = import sources.nixpkgs { + overlays = [ + overlays.dataplane + ]; + }; in { From 2d964c060588371bcc889838f344b027635acfae Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 01:50:19 +0000 Subject: [PATCH 28/38] build(nix): pipe env into overlays We don't yet have the machinery to generate our compile flags, but we can pre-emptively wire up the system which will feed in those flags into our overlay. Signed-off-by: Daniel Noland --- nix/overlays/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/overlays/default.nix b/nix/overlays/default.nix index f112df68c..6139dfb3c 100644 --- a/nix/overlays/default.nix +++ b/nix/overlays/default.nix @@ -1,9 +1,9 @@ { sources, + env ? { }, }: { dataplane = import ./dataplane.nix { - inherit sources; - env = { }; + inherit sources env; }; } From a3b02ee39b6ec9f6dba784101ae97057d0622e56 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 02:52:56 +0000 Subject: [PATCH 29/38] build(nix): frame common profile Now that we have our overlay in a place where it can accept flags we can start constructing machinery to generate those flags. We start with the "common" profile. This profile consists of flags we expect to pass to each and every build environment on all target platforms. Flags should be added to this profile conservatively, as they apply globally. Do not add performance compromising flags here. Signed-off-by: Daniel Noland --- profiles.nix | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 profiles.nix diff --git a/profiles.nix b/profiles.nix new file mode 100644 index 000000000..1dd895715 --- /dev/null +++ b/profiles.nix @@ -0,0 +1,20 @@ +let + common.NIX_CFLAGS_COMPILE = [ + "-glldb" + "-gdwarf-5" + # odr or strict-aliasing violations are indicative of LTO incompatibility, so check for that + "-Werror=odr" + "-Werror=strict-aliasing" + ]; + common.NIX_CXXFLAGS_COMPILE = common.NIX_CFLAGS_COMPILE; + common.NIX_CFLAGS_LINK = [ + # getting proper LTO from LLVM compiled objects is best done with lld rather than ld, mold, or wild (at least at the + # time of writing) + "-fuse-ld=lld" + # we always want pic/pie and GOT offsets should be computed at compile time whenever possible + "-Wl,-z,relro,-z,now" + ]; +in +{ + +} From ff79f7ccc7342cc7e297981f4cf49f5b622222df Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 02:53:19 +0000 Subject: [PATCH 30/38] build(nix): frame debug profile This commit constructs the "debug" profile. This profile should only be activated in environments where performance is of much less concern than debugging. Signed-off-by: Daniel Noland --- profiles.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/profiles.nix b/profiles.nix index 1dd895715..31d226f7b 100644 --- a/profiles.nix +++ b/profiles.nix @@ -14,6 +14,14 @@ let # we always want pic/pie and GOT offsets should be computed at compile time whenever possible "-Wl,-z,relro,-z,now" ]; + debug.NIX_CFLAGS_COMPILE = [ + "-fno-inline" + "-fno-omit-frame-pointer" + "-D_FORTIFY_SOURCE=0" # disable security stuff because the goal is to make the asm as easy to understand as possible + "-Wno-macro-redefined" # many apps opt in to _FORTIFY_SOURCE={1,2,3} explicitly, and -Wall errors when you redefine + ]; + debug.NIX_CXXFLAGS_COMPILE = debug.NIX_CFLAGS_COMPILE; + debug.NIX_CFLAGS_LINK = [ ]; in { From 3795117ee09d6b85cff78b93750406a008ac651d Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 02:53:52 +0000 Subject: [PATCH 31/38] build(nix): frame optimize profile The optimize profile contains flags which optimize the performance of the generated artifacts. This build profile makes no attempt to preserve debugability and will greatly increase build times when activated. Signed-off-by: Daniel Noland --- profiles.nix | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/profiles.nix b/profiles.nix index 31d226f7b..daa527d25 100644 --- a/profiles.nix +++ b/profiles.nix @@ -22,6 +22,21 @@ let ]; debug.NIX_CXXFLAGS_COMPILE = debug.NIX_CFLAGS_COMPILE; debug.NIX_CFLAGS_LINK = [ ]; + optimize.NIX_CFLAGS_COMPILE = [ + "-O3" + "-flto=full" + "-fsplit-lto-unit" # important for compatibility with rust's LTO + ]; + optimize.NIX_CXXFLAGS_COMPILE = optimize.NIX_CFLAGS_COMPILE ++ [ + "-fwhole-program-vtables" + ]; + optimize.NIX_CFLAGS_LINK = [ + "-flto=full" + "-Wl,--lto-whole-program-visibility" + # just to keep the artifacts small, we don't currently use any linked artifact anyway + "-Wl,--gc-sections" + "-Wl,--as-needed" + ]; in { From 3923d8217eb7fdc07b8a3cd762a1e73823509d9d Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 02:54:19 +0000 Subject: [PATCH 32/38] build(nix): frame secure profile The secure profile is a set of flags intended to degrade security vulnerabilities into denial of service attacks. This should mostly be used in release builds, but can be applied to debug or (especially) to (future) fuzz environments as a means of instructing the fuzzer to hunt bugs with plausible security impacts. In particular, enabling safe-stack and or cfi (both future goals) should be done in future fuzz builds if possible. Signed-off-by: Daniel Noland --- profiles.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/profiles.nix b/profiles.nix index daa527d25..0e4ee1ac1 100644 --- a/profiles.nix +++ b/profiles.nix @@ -37,6 +37,13 @@ let "-Wl,--gc-sections" "-Wl,--as-needed" ]; + secure.NIX_CFLAGS_COMPILE = [ + "-fstack-protector-strong" + "-fstack-clash-protection" + ]; + secure.NIX_CXXFLAGS_COMPILE = secure.NIX_CFLAGS_COMPILE; + # handing the CFLAGS back to clang/lld is basically required for -fsanitize + secure.NIX_CFLAGS_LINK = secure.NIX_CFLAGS_COMPILE; in { From e4a1287783f58ba88325af5dbb29c457fa0352d1 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 02:54:47 +0000 Subject: [PATCH 33/38] build(nix): add notes about cf-protection, cfi, and safe-stack Signed-off-by: Daniel Noland --- profiles.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/profiles.nix b/profiles.nix index 0e4ee1ac1..512a384a4 100644 --- a/profiles.nix +++ b/profiles.nix @@ -40,6 +40,13 @@ let secure.NIX_CFLAGS_COMPILE = [ "-fstack-protector-strong" "-fstack-clash-protection" + # "-fcf-protection=full" # requires extra testing before we enable + # "-fsanitize=safe-stack" # requires extra testing before we enable (not compatible with musl) + # "-fsanitize=cfi" # requires extra testing before we enable + # enable if you turn on cfi to properly link with rust + # "-fsanitize-cfi-icall-experimental-normalize-integers" + # consider enabling if you turn on cfi (not compatible with cross DSO cfi) + # "-fsanitize-cfi-icall-generalize-pointers" ]; secure.NIX_CXXFLAGS_COMPILE = secure.NIX_CFLAGS_COMPILE; # handing the CFLAGS back to clang/lld is basically required for -fsanitize From c48c8c429ba6eb22c4bbd5e7b6623127954eeeae Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 02:55:21 +0000 Subject: [PATCH 34/38] build(nix): intro combine-profiles This commit introduces a (regrettably opaque) helper function for composing build environments from an arbitrary combination of profiles. Signed-off-by: Daniel Noland --- profiles.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/profiles.nix b/profiles.nix index 512a384a4..633c236e7 100644 --- a/profiles.nix +++ b/profiles.nix @@ -51,6 +51,11 @@ let secure.NIX_CXXFLAGS_COMPILE = secure.NIX_CFLAGS_COMPILE; # handing the CFLAGS back to clang/lld is basically required for -fsanitize secure.NIX_CFLAGS_LINK = secure.NIX_CFLAGS_COMPILE; + combine-profiles = + features: + builtins.foldl' ( + acc: elem: builtins.mapAttrs (var: val: (acc.${var} or [ ]) ++ val) elem + ) { } features; in { From c78d7778e521f60e647352d07e0c986998a024d7 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 02:55:44 +0000 Subject: [PATCH 35/38] build(nix): define debug environment Combine the common and debug profiles to define the debug environment. Signed-off-by: Daniel Noland --- profiles.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/profiles.nix b/profiles.nix index 633c236e7..65b63a08d 100644 --- a/profiles.nix +++ b/profiles.nix @@ -58,5 +58,8 @@ let ) { } features; in { - + debug = combine-profiles [ + common + debug + ]; } From bfeb6838145e3ad8e55bdec4fa4644a2addbc8b4 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 02:56:03 +0000 Subject: [PATCH 36/38] build(nix): define release environment Combine the common, optimize, and secure profiles to define the release environment. Signed-off-by: Daniel Noland --- profiles.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/profiles.nix b/profiles.nix index 65b63a08d..5fcc49979 100644 --- a/profiles.nix +++ b/profiles.nix @@ -62,4 +62,9 @@ in common debug ]; + release = combine-profiles [ + common + optimize + secure + ]; } From 89cd6ee6cdc42682dbddcba6eb9d0127b96bfe53 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 02:57:29 +0000 Subject: [PATCH 37/38] build(nix): pipe environments into build This commit (somewhat awkwardly) injects our build environments as top level children of the pkgs key. I am aware that I can do nix + fp tricks to make this more succinct and flexible, but I really am trying to keep this nix code basic where practical, and I don't expect us to have a very large number of environments. Signed-off-by: Daniel Noland --- default.nix | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index ec67ab546..776d58bcf 100644 --- a/default.nix +++ b/default.nix @@ -1,11 +1,22 @@ let sources = import ./npins; - overlays = import ./nix/overlays { + profiles = import ./profiles.nix; + overlays.debug = import ./nix/overlays { inherit sources; + env = profiles.debug; }; - pkgs = import sources.nixpkgs { + overlays.release = import ./nix/overlays { + inherit sources; + env = profiles.release; + }; + pkgs.debug = import sources.nixpkgs { + overlays = [ + overlays.debug.dataplane + ]; + }; + pkgs.release = import sources.nixpkgs { overlays = [ - overlays.dataplane + overlays.release.dataplane ]; }; in From 9b8ac90fdd171eb339efd1437764bfa8294e21be Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Mon, 15 Dec 2025 23:07:23 +0000 Subject: [PATCH 38/38] build(nix): introduce fat-lto-objects flag (for the moment) This flag allows builds which make use of tools like nm to build when lto is enabled. We enable it here (for the moment) to allow rdma-core to build on aarch64 / aarch64-musl. That said, I don't like this flag as it makes the object files and thus the archive files much bigger. It opens the window for LTO linking mistakes which are much easier avoided if this is off. Once I have cooked a patch for rdma-core to skip the nm step we can revert this commit. Signed-off-by: Daniel Noland --- profiles.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/profiles.nix b/profiles.nix index 5fcc49979..b3d52d626 100644 --- a/profiles.nix +++ b/profiles.nix @@ -25,6 +25,7 @@ let optimize.NIX_CFLAGS_COMPILE = [ "-O3" "-flto=full" + "-ffat-lto-objects" "-fsplit-lto-unit" # important for compatibility with rust's LTO ]; optimize.NIX_CXXFLAGS_COMPILE = optimize.NIX_CFLAGS_COMPILE ++ [